Examples with TableId org.apache.accumulo.core.data.TableId used on opensource projects

Example 66 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class FateConcurrencyIT method lookupFateInZookeeper.

/**
 * Checks fates in zookeeper looking for transaction associated with a compaction as a double
 * check that the test will be valid because the running compaction does have a fate transaction
 * lock.
 * <p>
 * This method throws can throw either IllegalStateException (failed) or a Zookeeper exception.
 * Throwing the Zookeeper exception allows for retries if desired to handle transient zookeeper
 * issues.
 *
 * @param tableName
 *          a table name
 * @return true if corresponding fate transaction found, false otherwise
 * @throws KeeperException
 *           if a zookeeper error occurred - allows for retries.
 */
private boolean lookupFateInZookeeper(final String tableName) throws KeeperException {
    AdminUtil<String> admin = new AdminUtil<>(false);
    try {
        TableId tableId = context.getTableId(tableName);
        log.trace("tid: {}", tableId);
        InstanceId instanceId = context.getInstanceID();
        ZooReaderWriter zk = new ZooReaderWriter(context.getZooKeepers(), context.getZooKeepersSessionTimeOut(), secret);
        ZooStore<String> zs = new ZooStore<>(ZooUtil.getRoot(instanceId) + Constants.ZFATE, zk);
        AdminUtil.FateStatus fateStatus = admin.getStatus(zs, zk, ZooUtil.getRoot(instanceId) + Constants.ZTABLE_LOCKS + "/" + tableId, null, null);
        log.trace("current fates: {}", fateStatus.getTransactions().size());
        for (AdminUtil.TransactionStatus tx : fateStatus.getTransactions()) {
            if (isCompaction(tx))
                return true;
        }
    } catch (TableNotFoundException | InterruptedException ex) {
        throw new IllegalStateException(ex);
    }
    // did not find appropriate fate transaction for compaction.
    return Boolean.FALSE;
}

Example 67 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class FateConcurrencyIT method getFateStatus.

/**
 * Validate the the AdminUtil.getStatus works correctly after refactor and validate that
 * getTransactionStatus can be called without lock map(s). The test starts a long running fate
 * transaction (slow compaction) and the calls AdminUtil functions to get the FATE.
 */
@Test
public void getFateStatus() {
    SlowOps.setExpectedCompactions(client, 1);
    String tableName = getUniqueNames(1)[0];
    slowOps = new SlowOps(client, tableName, maxWaitMillis);
    TableId tableId;
    try {
        assertEquals("verify table online after created", TableState.ONLINE, getTableState(tableName));
        tableId = context.getTableId(tableName);
        log.trace("tid: {}", tableId);
    } catch (TableNotFoundException ex) {
        throw new IllegalStateException(String.format("Table %s does not exist, failing test", tableName));
    }
    slowOps.startCompactTask();
    AdminUtil.FateStatus withLocks = null;
    List<AdminUtil.TransactionStatus> noLocks = null;
    int maxRetries = 3;
    AdminUtil<String> admin = new AdminUtil<>(false);
    while (maxRetries > 0) {
        try {
            InstanceId instanceId = context.getInstanceID();
            ZooReaderWriter zk = new ZooReaderWriter(context.getZooKeepers(), context.getZooKeepersSessionTimeOut(), secret);
            ZooStore<String> zs = new ZooStore<>(ZooUtil.getRoot(instanceId) + Constants.ZFATE, zk);
            withLocks = admin.getStatus(zs, zk, ZooUtil.getRoot(instanceId) + Constants.ZTABLE_LOCKS + "/" + tableId, null, null);
            // call method that does not use locks.
            noLocks = admin.getTransactionStatus(zs, null, null);
            // no zk exception, no need to retry
            break;
        } catch (InterruptedException ex) {
            Thread.currentThread().interrupt();
            fail("Interrupt received - test failed");
            return;
        } catch (KeeperException ex) {
            maxRetries--;
            try {
                Thread.sleep(1000);
            } catch (InterruptedException intr_ex) {
                Thread.currentThread().interrupt();
                return;
            }
        }
    }
    assertNotNull(withLocks);
    assertNotNull(noLocks);
    // fast check - count number of transactions
    assertEquals(withLocks.getTransactions().size(), noLocks.size());
    int matchCount = 0;
    for (AdminUtil.TransactionStatus tx : withLocks.getTransactions()) {
        if (isCompaction(tx)) {
            log.trace("Fate id: {}, status: {}", tx.getTxid(), tx.getStatus());
            for (AdminUtil.TransactionStatus tx2 : noLocks) {
                if (tx2.getTxid().equals(tx.getTxid())) {
                    matchCount++;
                }
            }
        }
    }
    assertTrue("Number of fates matches should be > 0", matchCount > 0);
    try {
        // test complete, cancel compaction and move on.
        client.tableOperations().cancelCompaction(tableName);
        // block if compaction still running
        boolean cancelled = slowOps.blockWhileCompactionRunning();
        log.debug("Cancel completed successfully: {}", cancelled);
    } catch (TableNotFoundException | AccumuloSecurityException | AccumuloException ex) {
        log.debug("Could not cancel compaction due to exception", ex);
    }
}

Example 68 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class FunctionalTestUtils method getRFilePaths.

public static List<String> getRFilePaths(AccumuloClient c, String tableName) throws Exception {
    List<String> files = new ArrayList<>();
    try (Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        TableId tableId = TableId.of(c.tableOperations().tableIdMap().get(tableName));
        scanner.setRange(TabletsSection.getRange(tableId));
        scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
        scanner.forEach(entry -> {
            files.add(entry.getKey().getColumnQualifier().toString());
        });
    }
    return files;
}

Example 69 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class AccumuloRecordReader method getSplits.

public static List<InputSplit> getSplits(JobContext context, Class<?> callingClass) throws IOException {
    validateOptions(context, callingClass);
    LinkedList<InputSplit> splits = new LinkedList<>();
    try (AccumuloClient client = createClient(context, callingClass);
        var clientContext = ((ClientContext) client)) {
        Map<String, InputTableConfig> tableConfigs = InputConfigurator.getInputTableConfigs(callingClass, context.getConfiguration());
        for (Map.Entry<String, InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
            String tableName = tableConfigEntry.getKey();
            InputTableConfig tableConfig = tableConfigEntry.getValue();
            TableId tableId;
            // resolve table name to id once, and use id from this point forward
            try {
                tableId = clientContext.getTableId(tableName);
            } catch (TableNotFoundException e) {
                throw new IOException(e);
            }
            boolean batchScan = InputConfigurator.isBatchScan(callingClass, context.getConfiguration());
            boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
            if (batchScan && !supportBatchScan)
                throw new IllegalArgumentException("BatchScanner optimization not available for offline" + " scan, isolated, or local iterators");
            boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
            if (batchScan && !autoAdjust)
                throw new IllegalArgumentException("AutoAdjustRanges must be enabled when using BatchScanner optimization");
            List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
            if (ranges.isEmpty()) {
                ranges = new ArrayList<>(1);
                ranges.add(new Range());
            }
            // get the metadata information for these ranges
            Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>();
            TabletLocator tl;
            try {
                if (tableConfig.isOfflineScan()) {
                    binnedRanges = binOfflineTable(context, tableId, ranges, callingClass);
                    while (binnedRanges == null) {
                        // Some tablets were still online, try again
                        // sleep randomly between 100 and 200 ms
                        sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
                        binnedRanges = binOfflineTable(context, tableId, ranges, callingClass);
                    }
                } else {
                    tl = InputConfigurator.getTabletLocator(callingClass, context.getConfiguration(), tableId);
                    // its possible that the cache could contain complete, but old information about a
                    // tables tablets... so clear it
                    tl.invalidateCache();
                    while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) {
                        clientContext.requireNotDeleted(tableId);
                        clientContext.requireNotOffline(tableId, tableName);
                        binnedRanges.clear();
                        log.warn("Unable to locate bins for specified ranges. Retrying.");
                        // sleep randomly between 100 and 200 ms
                        sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
                        tl.invalidateCache();
                    }
                }
            } catch (TableOfflineException | TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
                throw new IOException(e);
            }
            // all of this code will add either range per each locations or split ranges and add
            // range-location split
            // Map from Range to Array of Locations, we only use this if we're don't split
            HashMap<Range, ArrayList<String>> splitsToAdd = null;
            if (!autoAdjust)
                splitsToAdd = new HashMap<>();
            HashMap<String, String> hostNameCache = new HashMap<>();
            for (Map.Entry<String, Map<KeyExtent, List<Range>>> tserverBin : binnedRanges.entrySet()) {
                String ip = tserverBin.getKey().split(":", 2)[0];
                String location = hostNameCache.get(ip);
                if (location == null) {
                    InetAddress inetAddress = InetAddress.getByName(ip);
                    location = inetAddress.getCanonicalHostName();
                    hostNameCache.put(ip, location);
                }
                for (Map.Entry<KeyExtent, List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
                    Range ke = extentRanges.getKey().toDataRange();
                    if (batchScan) {
                        // group ranges by tablet to be read by a BatchScanner
                        ArrayList<Range> clippedRanges = new ArrayList<>();
                        for (Range r : extentRanges.getValue()) clippedRanges.add(ke.clip(r));
                        BatchInputSplit split = new BatchInputSplit(tableName, tableId, clippedRanges, new String[] { location });
                        SplitUtils.updateSplit(split, tableConfig);
                        splits.add(split);
                    } else {
                        // not grouping by tablet
                        for (Range r : extentRanges.getValue()) {
                            if (autoAdjust) {
                                // divide ranges into smaller ranges, based on the tablets
                                RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonical(), ke.clip(r), new String[] { location });
                                SplitUtils.updateSplit(split, tableConfig);
                                split.setOffline(tableConfig.isOfflineScan());
                                split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
                                split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
                                splits.add(split);
                            } else {
                                // don't divide ranges
                                ArrayList<String> locations = splitsToAdd.get(r);
                                if (locations == null)
                                    locations = new ArrayList<>(1);
                                locations.add(location);
                                splitsToAdd.put(r, locations);
                            }
                        }
                    }
                }
            }
            if (!autoAdjust)
                for (Map.Entry<Range, ArrayList<String>> entry : splitsToAdd.entrySet()) {
                    RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonical(), entry.getKey(), entry.getValue().toArray(new String[0]));
                    SplitUtils.updateSplit(split, tableConfig);
                    split.setOffline(tableConfig.isOfflineScan());
                    split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
                    split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
                    splits.add(split);
                }
        }
    }
    return splits;
}

Example 70 with TableId

use of org.apache.accumulo.core.data.TableId in project accumulo by apache.

the class HostRegexTableLoadBalancerReconfigurationTest method testConfigurationChanges.

@Test
public void testConfigurationChanges() {
    HashMap<String, TableId> tables = new HashMap<>();
    tables.put(FOO.getTableName(), FOO.getId());
    tables.put(BAR.getTableName(), BAR.getId());
    tables.put(BAZ.getTableName(), BAZ.getId());
    ConfigurationCopy config = new ConfigurationCopy(SiteConfiguration.auto());
    DEFAULT_TABLE_PROPERTIES.forEach(config::set);
    ConfigurationImpl configImpl = new ConfigurationImpl(config);
    BalancerEnvironment environment = createMock(BalancerEnvironment.class);
    expect(environment.getConfiguration()).andReturn(configImpl).anyTimes();
    expect(environment.getTableIdMap()).andReturn(tables).anyTimes();
    expect(environment.getConfiguration(anyObject(TableId.class))).andReturn(configImpl).anyTimes();
    replay(environment);
    init(environment);
    Map<TabletId, TabletServerId> unassigned = new HashMap<>();
    for (List<TabletId> tablets : tableTablets.values()) {
        for (TabletId tablet : tablets) {
            unassigned.put(tablet, null);
        }
    }
    this.getAssignments(new AssignmentParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments));
    assertEquals(15, assignments.size());
    // Ensure unique tservers
    for (Entry<TabletId, TabletServerId> e : assignments.entrySet()) {
        for (Entry<TabletId, TabletServerId> e2 : assignments.entrySet()) {
            if (e.getKey().equals(e2.getKey())) {
                continue;
            }
            if (e.getValue().equals(e2.getValue())) {
                fail("Assignment failure. " + e.getKey() + " and " + e2.getKey() + " are assigned to the same host: " + e.getValue());
            }
        }
    }
    // Ensure assignments are correct
    for (Entry<TabletId, TabletServerId> e : assignments.entrySet()) {
        if (!tabletInBounds(e.getKey(), e.getValue())) {
            fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
        }
    }
    Set<TabletId> migrations = new HashSet<>();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    // Wait to trigger the out of bounds check which will call our version of
    // getOnlineTabletsForTable
    UtilWaitThread.sleep(3000);
    this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut));
    assertEquals(0, migrationsOut.size());
    // Change property, simulate call by TableConfWatcher
    config.set(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*");
    // Wait to trigger the out of bounds check and the repool check
    UtilWaitThread.sleep(10000);
    this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut));
    assertEquals(5, migrationsOut.size());
    for (TabletMigration migration : migrationsOut) {
        assertTrue(migration.getNewTabletServer().getHost().startsWith("192.168.0.1") || migration.getNewTabletServer().getHost().startsWith("192.168.0.2") || migration.getNewTabletServer().getHost().startsWith("192.168.0.3") || migration.getNewTabletServer().getHost().startsWith("192.168.0.4") || migration.getNewTabletServer().getHost().startsWith("192.168.0.5"));
    }
}