Examples with KeyExtent org.apache.accumulo.core.dataImpl.KeyExtent used on opensource projects

Example 86 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class TableIT method test.

@Test
public void test() throws Exception {
    assumeTrue(getClusterType() == ClusterType.MINI);
    AccumuloCluster cluster = getCluster();
    MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster;
    String rootPath = mac.getConfig().getDir().getAbsolutePath();
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        TableOperations to = c.tableOperations();
        String tableName = getUniqueNames(1)[0];
        to.create(tableName);
        VerifyParams params = new VerifyParams(getClientProps(), tableName);
        TestIngest.ingest(c, params);
        to.flush(tableName, null, null, true);
        VerifyIngest.verifyIngest(c, params);
        TableId id = TableId.of(to.tableIdMap().get(tableName));
        try (Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            s.setRange(new KeyExtent(id, null, null).toMetaRange());
            s.fetchColumnFamily(DataFileColumnFamily.NAME);
            assertTrue(Iterators.size(s.iterator()) > 0);
            FileSystem fs = getCluster().getFileSystem();
            assertTrue(fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)).length > 0);
            to.delete(tableName);
            assertEquals(0, Iterators.size(s.iterator()));
            try {
                assertEquals(0, fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)).length);
            } catch (FileNotFoundException ex) {
            // that's fine, too
            }
            assertNull(to.tableIdMap().get(tableName));
            to.create(tableName);
            TestIngest.ingest(c, params);
            VerifyIngest.verifyIngest(c, params);
            to.delete(tableName);
        }
    }
}

Example 87 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class WALSunnyDayIT method test.

@Test
public void test() throws Exception {
    MiniAccumuloClusterImpl mac = getCluster();
    MiniAccumuloClusterControl control = mac.getClusterControl();
    control.stop(GARBAGE_COLLECTOR);
    ServerContext context = getServerContext();
    try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
        String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        writeSomeData(c, tableName, 1, 1);
        // wal markers are added lazily
        Map<String, WalState> wals = getWALsAndAssertCount(context, 2);
        assertEquals("all WALs should be in use", 2, countInUse(wals.values()));
        // roll log, get a new next
        writeSomeData(c, tableName, 1001, 50);
        Map<String, WalState> walsAfterRoll = getWALsAndAssertCount(context, 3);
        assertTrue("new WALs should be a superset of the old WALs", walsAfterRoll.keySet().containsAll(wals.keySet()));
        assertEquals("all WALs should be in use", 3, countInUse(walsAfterRoll.values()));
        // flush the tables
        for (String table : new String[] { tableName, MetadataTable.NAME, RootTable.NAME }) {
            c.tableOperations().flush(table, null, null, true);
        }
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        // rolled WAL is no longer in use, but needs to be GC'd
        Map<String, WalState> walsAfterflush = getWALsAndAssertCount(context, 3);
        assertEquals("inUse should be 2", 2, countInUse(walsAfterflush.values()));
        // let the GC run for a little bit
        control.start(GARBAGE_COLLECTOR);
        sleepUninterruptibly(5, TimeUnit.SECONDS);
        // make sure the unused WAL goes away
        getWALsAndAssertCount(context, 2);
        control.stop(GARBAGE_COLLECTOR);
        // restart the tserver, but don't run recovery on all tablets
        control.stop(TABLET_SERVER);
        // this delays recovery on the normal tables
        assertEquals(0, cluster.exec(SetGoalState.class, "SAFE_MODE").getProcess().waitFor());
        control.start(TABLET_SERVER);
        // wait for the metadata table to go back online
        getRecoveryMarkers(c);
        // allow a little time for the manager to notice ASSIGNED_TO_DEAD_SERVER tablets
        sleepUninterruptibly(5, TimeUnit.SECONDS);
        Map<KeyExtent, List<String>> markers = getRecoveryMarkers(c);
        // log.debug("markers " + markers);
        assertEquals("one tablet should have markers", 1, markers.size());
        assertEquals("tableId of the keyExtent should be 1", "1", markers.keySet().iterator().next().tableId().canonical());
        // put some data in the WAL
        assertEquals(0, cluster.exec(SetGoalState.class, "NORMAL").getProcess().waitFor());
        verifySomeData(c, tableName, 1001 * 50 + 1);
        writeSomeData(c, tableName, 100, 100);
        Map<String, WalState> walsAfterRestart = getWALsAndAssertCount(context, 4);
        // log.debug("wals after " + walsAfterRestart);
        assertEquals("used WALs after restart should be 4", 4, countInUse(walsAfterRestart.values()));
        control.start(GARBAGE_COLLECTOR);
        sleepUninterruptibly(5, TimeUnit.SECONDS);
        Map<String, WalState> walsAfterRestartAndGC = getWALsAndAssertCount(context, 2);
        assertEquals("logs in use should be 2", 2, countInUse(walsAfterRestartAndGC.values()));
    }
}

Example 88 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class ManagerClientServiceHandler method reportSplitExtent.

@Override
public void reportSplitExtent(TInfo info, TCredentials credentials, String serverName, TabletSplit split) {
    KeyExtent oldTablet = KeyExtent.fromThrift(split.oldTablet);
    if (manager.migrations.remove(oldTablet) != null) {
        Manager.log.info("Canceled migration of {}", split.oldTablet);
    }
    for (TServerInstance instance : manager.tserverSet.getCurrentServers()) {
        if (serverName.equals(instance.getHostPort())) {
            manager.nextEvent.event("%s reported split %s, %s", serverName, KeyExtent.fromThrift(split.newTablets.get(0)), KeyExtent.fromThrift(split.newTablets.get(1)));
            return;
        }
    }
    Manager.log.warn("Got a split from a server we don't recognize: {}", serverName);
}

Example 89 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class CompactionFinalizer method processPending.

private void processPending() {
    while (!Thread.interrupted()) {
        try {
            ArrayList<ExternalCompactionFinalState> batch = new ArrayList<>();
            batch.add(pendingNotifications.take());
            pendingNotifications.drainTo(batch);
            List<Future<?>> futures = new ArrayList<>();
            List<ExternalCompactionId> statusesToDelete = new ArrayList<>();
            Map<KeyExtent, TabletMetadata> tabletsMetadata;
            var extents = batch.stream().map(ExternalCompactionFinalState::getExtent).collect(toList());
            try (TabletsMetadata tablets = context.getAmple().readTablets().forTablets(extents).fetch(ColumnType.LOCATION, ColumnType.PREV_ROW, ColumnType.ECOMP).build()) {
                tabletsMetadata = tablets.stream().collect(toMap(TabletMetadata::getExtent, identity()));
            }
            for (ExternalCompactionFinalState ecfs : batch) {
                TabletMetadata tabletMetadata = tabletsMetadata.get(ecfs.getExtent());
                if (tabletMetadata == null || !tabletMetadata.getExternalCompactions().keySet().contains(ecfs.getExternalCompactionId())) {
                    // there is not per tablet external compaction entry, so delete its final state marker
                    // from metadata table
                    LOG.debug("Unable to find tablets external compaction entry, deleting completion entry {}", ecfs);
                    statusesToDelete.add(ecfs.getExternalCompactionId());
                } else if (tabletMetadata.getLocation() != null && tabletMetadata.getLocation().getType() == LocationType.CURRENT) {
                    futures.add(ntfyExecutor.submit(() -> notifyTserver(tabletMetadata.getLocation(), ecfs)));
                } else {
                    LOG.trace("External compaction {} is completed, but there is no location for tablet.  Unable to notify tablet, will try again later.", ecfs);
                }
            }
            if (!statusesToDelete.isEmpty()) {
                LOG.info("Deleting unresolvable completed external compactions from metadata table, ids: {}", statusesToDelete);
                context.getAmple().deleteExternalCompactionFinalStates(statusesToDelete);
            }
            for (Future<?> future : futures) {
                try {
                    future.get();
                } catch (ExecutionException e) {
                    LOG.debug("Failed to notify tserver", e);
                }
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new RuntimeException(e);
        } catch (RuntimeException e) {
            LOG.warn("Failed to process pending notifications", e);
        }
    }
}

Example 90 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class ThriftClientHandler method flush.

@Override
public void flush(TInfo tinfo, TCredentials credentials, String lock, String tableId, ByteBuffer startRow, ByteBuffer endRow) {
    try {
        checkPermission(credentials, lock, "flush");
    } catch (ThriftSecurityException e) {
        log.error("Caller doesn't have permission to flush a table", e);
        throw new RuntimeException(e);
    }
    ArrayList<Tablet> tabletsToFlush = new ArrayList<>();
    KeyExtent ke = new KeyExtent(TableId.of(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow));
    for (Tablet tablet : server.getOnlineTablets().values()) {
        if (ke.overlaps(tablet.getExtent())) {
            tabletsToFlush.add(tablet);
        }
    }
    Long flushID = null;
    for (Tablet tablet : tabletsToFlush) {
        if (flushID == null) {
            // it for each tablet
            try {
                flushID = tablet.getFlushID();
            } catch (NoNodeException e) {
                // table was probably deleted
                log.info("Asked to flush table that has no flush id {} {}", ke, e.getMessage());
                return;
            }
        }
        tablet.flush(flushID);
    }
}