Examples with TableId org.apache.accumulo.core.data.TableId used on opensource projects

Example 31 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class ExternalCompaction_3_IT method testMergeCancelsExternalCompaction.

@Test
public void testMergeCancelsExternalCompaction() throws Exception {
    getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
    getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE1);
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table1, "cs1", 2);
        // set compaction ratio to 1 so that majc occurs naturally, not user compaction
        // user compaction blocks merge
        client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.toString(), "1.0");
        // cause multiple rfiles to be created
        writeData(client, table1);
        writeData(client, table1);
        writeData(client, table1);
        writeData(client, table1);
        TableId tid = getCluster().getServerContext().getTableId(table1);
        // Wait for the compaction to start by waiting for 1 external compaction column
        Set<ExternalCompactionId> ecids = waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
        var md = new ArrayList<TabletMetadata>();
        try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.PREV_ROW).build()) {
            tm.forEach(t -> md.add(t));
            assertEquals(2, md.size());
        }
        // Merge - blocking operation
        Text start = md.get(0).getPrevEndRow();
        Text end = md.get(1).getEndRow();
        client.tableOperations().merge(table1, start, end);
        confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
        // ensure compaction ids were deleted by merge operation from metadata table
        try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build()) {
            Set<ExternalCompactionId> ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
            // keep checking until test times out
            while (!Collections.disjoint(ecids, ecids2)) {
                UtilWaitThread.sleep(25);
                ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
            }
        }
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().delete(table1);
    }
}

Example 32 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class GarbageCollectorCommunicatesWithTServersIT method getFilesForTable.

/**
 * Fetch all of the rfiles referenced by tablets in the metadata table for this table
 */
private Set<String> getFilesForTable(String tableName) throws Exception {
    final AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build();
    final TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
    assertNotNull("Could not determine table ID for " + tableName, tableId);
    Set<String> rfiles = new HashSet<>();
    try (Scanner s = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        Range r = TabletsSection.getRange(tableId);
        s.setRange(r);
        s.fetchColumnFamily(DataFileColumnFamily.NAME);
        for (Entry<Key, Value> entry : s) {
            log.debug("Reading RFiles: {}={}", entry.getKey().toStringNoTruncate(), entry.getValue());
            // uri://path/to/wal
            String cq = entry.getKey().getColumnQualifier().toString();
            String path = new Path(cq).toString();
            log.debug("Normalize path to rfile: {}", path);
            rfiles.add(path);
        }
    }
    return rfiles;
}

Example 33 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class WorkMakerIT method singleUnitMultipleTargets.

@Test
public void singleUnitMultipleTargets() throws Exception {
    String table = testName.getMethodName();
    client.tableOperations().create(table);
    TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table));
    String file = "hdfs://localhost:8020/accumulo/wal/123456-1234-1234-12345678";
    Mutation m = new Mutation(new Path(file).toString());
    m.put(StatusSection.NAME, new Text(tableId.canonical()), StatusUtil.fileCreatedValue(System.currentTimeMillis()));
    BatchWriter bw = ReplicationTable.getBatchWriter(client);
    bw.addMutation(m);
    bw.flush();
    // Assert that we have one record in the status section
    Set<ReplicationTarget> expectedTargets = new HashSet<>();
    try (Scanner s = ReplicationTable.getScanner(client)) {
        StatusSection.limit(s);
        assertEquals(1, Iterables.size(s));
        MockWorkMaker workMaker = new MockWorkMaker(client);
        Map<String, String> targetClusters = Map.of("remote_cluster_1", "4", "remote_cluster_2", "6", "remote_cluster_3", "8");
        for (Entry<String, String> cluster : targetClusters.entrySet()) {
            expectedTargets.add(new ReplicationTarget(cluster.getKey(), cluster.getValue(), tableId));
        }
        workMaker.setBatchWriter(bw);
        workMaker.addWorkRecord(new Text(file), StatusUtil.fileCreatedValue(System.currentTimeMillis()), targetClusters, tableId);
    }
    try (Scanner s = ReplicationTable.getScanner(client)) {
        WorkSection.limit(s);
        Set<ReplicationTarget> actualTargets = new HashSet<>();
        for (Entry<Key, Value> entry : s) {
            assertEquals(file, entry.getKey().getRow().toString());
            assertEquals(WorkSection.NAME, entry.getKey().getColumnFamily());
            ReplicationTarget target = ReplicationTarget.from(entry.getKey().getColumnQualifier());
            actualTargets.add(target);
        }
        for (ReplicationTarget expected : expectedTargets) {
            assertTrue("Did not find expected target: " + expected, actualTargets.contains(expected));
            actualTargets.remove(expected);
        }
        assertTrue("Found extra replication work entries: " + actualTargets, actualTargets.isEmpty());
    }
}

Example 34 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class ReplicationIT method replicationEntriesPrecludeWalDeletion.

@Test
public void replicationEntriesPrecludeWalDeletion() throws Exception {
    final ServerContext context = getServerContext();
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String table1 = "table1", table2 = "table2", table3 = "table3";
        final Multimap<String, TableId> logs = HashMultimap.create();
        final AtomicBoolean keepRunning = new AtomicBoolean(true);
        Thread t = new Thread(() -> {
            // when that happens
            while (keepRunning.get()) {
                try {
                    logs.putAll(getAllLogs(client, context));
                } catch (Exception e) {
                    log.error("Error getting logs", e);
                }
            }
        });
        t.start();
        HashMap<String, String> replicate_props = new HashMap<>();
        replicate_props.put(Property.TABLE_REPLICATION.getKey(), "true");
        replicate_props.put(Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
        client.tableOperations().create(table1, new NewTableConfiguration().setProperties(replicate_props));
        Thread.sleep(2000);
        // Write some data to table1
        writeSomeData(client, table1, 200, 500);
        client.tableOperations().create(table2, new NewTableConfiguration().setProperties(replicate_props));
        Thread.sleep(2000);
        writeSomeData(client, table2, 200, 500);
        client.tableOperations().create(table3, new NewTableConfiguration().setProperties(replicate_props));
        Thread.sleep(2000);
        writeSomeData(client, table3, 200, 500);
        // Force a write to metadata for the data written
        for (String table : Arrays.asList(table1, table2, table3)) {
            client.tableOperations().flush(table, null, null, true);
        }
        keepRunning.set(false);
        t.join(5000);
        // The manager is only running every second to create records in the replication table from
        // the
        // metadata table
        // Sleep a sufficient amount of time to ensure that we get the straggling WALs that might have
        // been created at the end
        Thread.sleep(5000);
        Set<String> replFiles = getReferencesToFilesToBeReplicated(client);
        // We might have a WAL that was use solely for the replication table
        // We want to remove that from our list as it should not appear in the replication table
        String replicationTableId = client.tableOperations().tableIdMap().get(ReplicationTable.NAME);
        Iterator<Entry<String, TableId>> observedLogs = logs.entries().iterator();
        while (observedLogs.hasNext()) {
            Entry<String, TableId> observedLog = observedLogs.next();
            if (replicationTableId.equals(observedLog.getValue().canonical())) {
                log.info("Removing {} because its tableId is for the replication table", observedLog);
                observedLogs.remove();
            }
        }
        // We should have *some* reference to each log that was seen in the metadata table
        // They might not yet all be closed though (might be newfile)
        assertTrue("Metadata log distribution: " + logs + "replFiles " + replFiles, logs.keySet().containsAll(replFiles));
        assertTrue("Difference between replication entries and current logs is bigger than one", logs.keySet().size() - replFiles.size() <= 1);
        final Configuration conf = new Configuration();
        for (String replFile : replFiles) {
            Path p = new Path(replFile);
            FileSystem fs = p.getFileSystem(conf);
            if (!fs.exists(p)) {
                // double-check: the garbage collector can be fast
                Set<String> currentSet = getReferencesToFilesToBeReplicated(client);
                log.info("Current references {}", currentSet);
                log.info("Looking for reference to {}", replFile);
                log.info("Contains? {}", currentSet.contains(replFile));
                assertTrue("File does not exist anymore, it was likely incorrectly garbage collected: " + p, !currentSet.contains(replFile));
            }
        }
    }
}

Example 35 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class ReplicationIT method correctClusterNameInWorkEntry.

@Test
public void correctClusterNameInWorkEntry() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String table1 = "table1";
        // replication shouldn't be online when we begin
        assertFalse(ReplicationTable.isOnline(client));
        // Create two tables
        client.tableOperations().create(table1);
        int attempts = 5;
        while (attempts > 0) {
            try {
                // Enable replication on table1
                client.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
                // Replicate table1 to cluster1 in the table with id of '4'
                client.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "4");
                attempts = 0;
            } catch (Exception e) {
                attempts--;
                if (attempts <= 0) {
                    throw e;
                }
                sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
            }
        }
        // Write some data to table1
        writeSomeData(client, table1, 2000, 50);
        client.tableOperations().flush(table1, null, null, true);
        TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table1));
        assertNotNull("Table ID was null", tableId);
        // Make sure the replication table exists at this point
        while (!ReplicationTable.isOnline(client)) {
            sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
        }
        assertTrue("Replication table did not exist", ReplicationTable.isOnline(client));
        for (int i = 0; i < 5 && !client.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ); i++) {
            Thread.sleep(1000);
        }
        assertTrue(client.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ));
        boolean notFound = true;
        for (int i = 0; i < 10 && notFound; i++) {
            try (Scanner s = ReplicationTable.getScanner(client)) {
                WorkSection.limit(s);
                try {
                    Entry<Key, Value> e = Iterables.getOnlyElement(s);
                    Text expectedColqual = new ReplicationTarget("cluster1", "4", tableId).toText();
                    assertEquals(expectedColqual, e.getKey().getColumnQualifier());
                    notFound = false;
                } catch (NoSuchElementException e) {
                } catch (IllegalArgumentException e) {
                    try (Scanner s2 = ReplicationTable.getScanner(client)) {
                        for (Entry<Key, Value> content : s2) {
                            log.info("{} => {}", content.getKey().toStringNoTruncate(), content.getValue());
                        }
                        fail("Found more than one work section entry");
                    }
                }
                Thread.sleep(500);
            }
        }
        if (notFound) {
            try (Scanner s = ReplicationTable.getScanner(client)) {
                for (Entry<Key, Value> content : s) {
                    log.info("{} => {}", content.getKey().toStringNoTruncate(), content.getValue());
                }
                assertFalse("Did not find the work entry for the status entry", notFound);
            }
        }
    }
}