Search in sources :

Example 1 with WalMarkerException

use of org.apache.accumulo.server.log.WalStateManager.WalMarkerException in project accumulo by apache.

the class VolumeIT method verifyVolumesUsed.

private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws Exception {
    Connector conn = getConnector();
    List<String> expected = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        String row = String.format("%06d", i * 100 + 3);
        expected.add(row + ":cf1:cq1:1");
    }
    if (!conn.tableOperations().exists(tableName)) {
        Assert.assertFalse(shouldExist);
        writeData(tableName, conn);
        verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
        conn.tableOperations().flush(tableName, null, null, true);
    }
    verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
    Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get(tableName));
    try (Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(metaScanner);
        metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
        metaScanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
        int[] counts = new int[paths.length];
        outer: for (Entry<Key, Value> entry : metaScanner) {
            String cf = entry.getKey().getColumnFamily().toString();
            String cq = entry.getKey().getColumnQualifier().toString();
            String path;
            if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME.toString()))
                path = cq;
            else
                path = entry.getValue().toString();
            for (int i = 0; i < paths.length; i++) {
                if (path.startsWith(paths[i].toString())) {
                    counts[i]++;
                    continue outer;
                }
            }
            Assert.fail("Unexpected volume " + path);
        }
        // keep retrying until WAL state information in ZooKeeper stabilizes or until test times out
        retry: while (true) {
            Instance i = conn.getInstance();
            ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
            WalStateManager wals = new WalStateManager(i, zk);
            try {
                outer: for (Entry<Path, WalState> entry : wals.getAllState().entrySet()) {
                    for (Path path : paths) {
                        if (entry.getKey().toString().startsWith(path.toString())) {
                            continue outer;
                        }
                    }
                    log.warn("Unexpected volume " + entry.getKey() + " (" + entry.getValue() + ")");
                    continue retry;
                }
            } catch (WalMarkerException e) {
                Throwable cause = e.getCause();
                if (cause instanceof NoNodeException) {
                    // ignore WALs being cleaned up
                    continue retry;
                }
                throw e;
            }
            break;
        }
        // if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
        // 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2 volumes would be chosen is 2.46e-18
        int sum = 0;
        for (int count : counts) {
            Assert.assertTrue(count > 0);
            sum += count;
        }
        Assert.assertEquals(200, sum);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) Connector(org.apache.accumulo.core.client.Connector) Scanner(org.apache.accumulo.core.client.Scanner) MetadataTable(org.apache.accumulo.core.metadata.MetadataTable) RootTable(org.apache.accumulo.core.metadata.RootTable) Table(org.apache.accumulo.core.client.impl.Table) NoNodeException(org.apache.zookeeper.KeeperException.NoNodeException) Instance(org.apache.accumulo.core.client.Instance) ZooKeeperInstance(org.apache.accumulo.core.client.ZooKeeperInstance) ArrayList(java.util.ArrayList) ZooReaderWriter(org.apache.accumulo.server.zookeeper.ZooReaderWriter) KeyExtent(org.apache.accumulo.core.data.impl.KeyExtent) Entry(java.util.Map.Entry) WalStateManager(org.apache.accumulo.server.log.WalStateManager) WalState(org.apache.accumulo.server.log.WalStateManager.WalState) WalMarkerException(org.apache.accumulo.server.log.WalStateManager.WalMarkerException)

Example 2 with WalMarkerException

use of org.apache.accumulo.server.log.WalStateManager.WalMarkerException in project accumulo by apache.

the class CloseWriteAheadLogReferences method getClosedLogs.

/**
 * Construct the set of referenced WALs from zookeeper
 *
 * @param conn
 *          Connector
 * @return The Set of WALs that are referenced in the metadata table
 */
protected HashSet<String> getClosedLogs(Connector conn) {
    WalStateManager wals = new WalStateManager(conn.getInstance(), ZooReaderWriter.getInstance());
    HashSet<String> result = new HashSet<>();
    try {
        for (Entry<Path, WalState> entry : wals.getAllState().entrySet()) {
            if (entry.getValue() == WalState.UNREFERENCED || entry.getValue() == WalState.CLOSED) {
                Path path = entry.getKey();
                log.debug("Found closed WAL " + path.toString());
                result.add(path.toString());
            }
        }
    } catch (WalMarkerException e) {
        throw new RuntimeException(e);
    }
    return result;
}
Also used : Path(org.apache.hadoop.fs.Path) WalStateManager(org.apache.accumulo.server.log.WalStateManager) WalState(org.apache.accumulo.server.log.WalStateManager.WalState) HashSet(java.util.HashSet) WalMarkerException(org.apache.accumulo.server.log.WalStateManager.WalMarkerException)

Example 3 with WalMarkerException

use of org.apache.accumulo.server.log.WalStateManager.WalMarkerException in project accumulo by apache.

the class TabletServer method markUnusedWALs.

private void markUnusedWALs() {
    Set<DfsLogger> candidates;
    synchronized (closedLogs) {
        candidates = new HashSet<>(closedLogs);
    }
    for (Tablet tablet : getOnlineTablets()) {
        candidates.removeAll(tablet.getCurrentLogFiles());
    }
    try {
        TServerInstance session = this.getTabletSession();
        for (DfsLogger candidate : candidates) {
            log.info("Marking " + candidate.getPath() + " as unreferenced");
            walMarker.walUnreferenced(session, candidate.getPath());
        }
        synchronized (closedLogs) {
            closedLogs.removeAll(candidates);
        }
    } catch (WalMarkerException ex) {
        log.info(ex.toString(), ex);
    }
}
Also used : Tablet(org.apache.accumulo.tserver.tablet.Tablet) TServerInstance(org.apache.accumulo.server.master.state.TServerInstance) DfsLogger(org.apache.accumulo.tserver.log.DfsLogger) WalMarkerException(org.apache.accumulo.server.log.WalStateManager.WalMarkerException)

Aggregations

WalMarkerException (org.apache.accumulo.server.log.WalStateManager.WalMarkerException)3 WalStateManager (org.apache.accumulo.server.log.WalStateManager)2 WalState (org.apache.accumulo.server.log.WalStateManager.WalState)2 Path (org.apache.hadoop.fs.Path)2 ArrayList (java.util.ArrayList)1 HashSet (java.util.HashSet)1 Entry (java.util.Map.Entry)1 Connector (org.apache.accumulo.core.client.Connector)1 Instance (org.apache.accumulo.core.client.Instance)1 Scanner (org.apache.accumulo.core.client.Scanner)1 ZooKeeperInstance (org.apache.accumulo.core.client.ZooKeeperInstance)1 Table (org.apache.accumulo.core.client.impl.Table)1 KeyExtent (org.apache.accumulo.core.data.impl.KeyExtent)1 MetadataTable (org.apache.accumulo.core.metadata.MetadataTable)1 RootTable (org.apache.accumulo.core.metadata.RootTable)1 TServerInstance (org.apache.accumulo.server.master.state.TServerInstance)1 ZooReaderWriter (org.apache.accumulo.server.zookeeper.ZooReaderWriter)1 DfsLogger (org.apache.accumulo.tserver.log.DfsLogger)1 Tablet (org.apache.accumulo.tserver.tablet.Tablet)1 NoNodeException (org.apache.zookeeper.KeeperException.NoNodeException)1