Examples with KeyExtent org.apache.accumulo.core.data.impl.KeyExtent used on opensource projects

Example 6 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class Master method update.

@Override
public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
    DeadServerList obit = new DeadServerList(ZooUtil.getRoot(getInstance()) + Constants.ZDEADTSERVERS);
    if (added.size() > 0) {
        log.info("New servers: {}", added);
        for (TServerInstance up : added) obit.delete(up.hostPort());
    }
    for (TServerInstance dead : deleted) {
        String cause = "unexpected failure";
        if (serversToShutdown.contains(dead))
            // maybe an incorrect assumption
            cause = "clean shutdown";
        if (!getMasterGoalState().equals(MasterGoalState.CLEAN_STOP))
            obit.post(dead.hostPort(), cause);
    }
    Set<TServerInstance> unexpected = new HashSet<>(deleted);
    unexpected.removeAll(this.serversToShutdown);
    if (unexpected.size() > 0) {
        if (stillMaster() && !getMasterGoalState().equals(MasterGoalState.CLEAN_STOP)) {
            log.warn("Lost servers {}", unexpected);
        }
    }
    serversToShutdown.removeAll(deleted);
    badServers.keySet().removeAll(deleted);
    // clear out any bad server with the same host/port as a new server
    synchronized (badServers) {
        cleanListByHostAndPort(badServers.keySet(), deleted, added);
    }
    synchronized (serversToShutdown) {
        cleanListByHostAndPort(serversToShutdown, deleted, added);
    }
    synchronized (migrations) {
        Iterator<Entry<KeyExtent, TServerInstance>> iter = migrations.entrySet().iterator();
        while (iter.hasNext()) {
            Entry<KeyExtent, TServerInstance> entry = iter.next();
            if (deleted.contains(entry.getValue())) {
                log.info("Canceling migration of {} to {}", entry.getKey(), entry.getValue());
                iter.remove();
            }
        }
    }
    nextEvent.event("There are now %d tablet servers", current.size());
}

Example 7 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class MasterClientServiceHandler method reportSplitExtent.

@Override
public void reportSplitExtent(TInfo info, TCredentials credentials, String serverName, TabletSplit split) {
    KeyExtent oldTablet = new KeyExtent(split.oldTablet);
    if (master.migrations.remove(oldTablet) != null) {
        Master.log.info("Canceled migration of {}", split.oldTablet);
    }
    for (TServerInstance instance : master.tserverSet.getCurrentServers()) {
        if (serverName.equals(instance.hostPort())) {
            master.nextEvent.event("%s reported split %s, %s", serverName, new KeyExtent(split.newTablets.get(0)), new KeyExtent(split.newTablets.get(1)));
            return;
        }
    }
    Master.log.warn("Got a split from a server we don't recognize: {}", serverName);
}

Example 8 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class MasterClientServiceHandler method waitForFlush.

@Override
public void waitForFlush(TInfo tinfo, TCredentials c, String tableIdStr, ByteBuffer startRow, ByteBuffer endRow, long flushID, long maxLoops) throws ThriftSecurityException, ThriftTableOperationException {
    Table.ID tableId = Table.ID.of(tableIdStr);
    Namespace.ID namespaceId = getNamespaceIdFromTableId(TableOperation.FLUSH, tableId);
    master.security.canFlush(c, tableId, namespaceId);
    if (endRow != null && startRow != null && ByteBufferUtil.toText(startRow).compareTo(ByteBufferUtil.toText(endRow)) >= 0)
        throw new ThriftTableOperationException(tableId.canonicalID(), null, TableOperation.FLUSH, TableOperationExceptionType.BAD_RANGE, "start row must be less than end row");
    Set<TServerInstance> serversToFlush = new HashSet<>(master.tserverSet.getCurrentServers());
    for (long l = 0; l < maxLoops; l++) {
        for (TServerInstance instance : serversToFlush) {
            try {
                final TServerConnection server = master.tserverSet.getConnection(instance);
                if (server != null)
                    server.flush(master.masterLock, tableId, ByteBufferUtil.toBytes(startRow), ByteBufferUtil.toBytes(endRow));
            } catch (TException ex) {
                Master.log.error(ex.toString());
            }
        }
        if (l == maxLoops - 1)
            break;
        sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
        serversToFlush.clear();
        try {
            Connector conn = master.getConnector();
            Scanner scanner;
            if (tableId.equals(MetadataTable.ID)) {
                scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
                scanner.setRange(MetadataSchema.TabletsSection.getRange());
            } else {
                scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
                Range range = new KeyExtent(tableId, null, ByteBufferUtil.toText(startRow)).toMetadataRange();
                scanner.setRange(range.clip(MetadataSchema.TabletsSection.getRange()));
            }
            TabletsSection.ServerColumnFamily.FLUSH_COLUMN.fetch(scanner);
            TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
            scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
            scanner.fetchColumnFamily(LogColumnFamily.NAME);
            RowIterator ri = new RowIterator(scanner);
            int tabletsToWaitFor = 0;
            int tabletCount = 0;
            Text ert = ByteBufferUtil.toText(endRow);
            while (ri.hasNext()) {
                Iterator<Entry<Key, Value>> row = ri.next();
                long tabletFlushID = -1;
                int logs = 0;
                boolean online = false;
                TServerInstance server = null;
                Entry<Key, Value> entry = null;
                while (row.hasNext()) {
                    entry = row.next();
                    Key key = entry.getKey();
                    if (TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier())) {
                        tabletFlushID = Long.parseLong(entry.getValue().toString());
                    }
                    if (LogColumnFamily.NAME.equals(key.getColumnFamily()))
                        logs++;
                    if (TabletsSection.CurrentLocationColumnFamily.NAME.equals(key.getColumnFamily())) {
                        online = true;
                        server = new TServerInstance(entry.getValue(), key.getColumnQualifier());
                    }
                }
                // when tablet is not online and has no logs, there is no reason to wait for it
                if ((online || logs > 0) && tabletFlushID < flushID) {
                    tabletsToWaitFor++;
                    if (server != null)
                        serversToFlush.add(server);
                }
                tabletCount++;
                Text tabletEndRow = new KeyExtent(entry.getKey().getRow(), (Text) null).getEndRow();
                if (tabletEndRow == null || (ert != null && tabletEndRow.compareTo(ert) >= 0))
                    break;
            }
            if (tabletsToWaitFor == 0)
                break;
            if (tabletCount == 0 && !Tables.exists(master.getInstance(), tableId))
                throw new ThriftTableOperationException(tableId.canonicalID(), null, TableOperation.FLUSH, TableOperationExceptionType.NOTFOUND, null);
        } catch (AccumuloException | TabletDeletedException e) {
            Master.log.debug("Failed to scan {} table to wait for flush {}", MetadataTable.NAME, tableId, e);
        } catch (AccumuloSecurityException e) {
            Master.log.warn("{}", e.getMessage(), e);
            throw new ThriftSecurityException();
        } catch (TableNotFoundException e) {
            Master.log.error("{}", e.getMessage(), e);
            throw new ThriftTableOperationException();
        }
    }
}

Example 9 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class ChaoticLoadBalancerTest method testAssignMigrations.

@Test
public void testAssignMigrations() {
    servers.clear();
    servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1234), "a"), new FakeTServer());
    servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1235), "b"), new FakeTServer());
    servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1236), "c"), new FakeTServer());
    Map<KeyExtent, TServerInstance> metadataTable = new TreeMap<>();
    String table = "t1";
    metadataTable.put(makeExtent(table, null, null), null);
    table = "t2";
    metadataTable.put(makeExtent(table, "a", null), null);
    metadataTable.put(makeExtent(table, null, "a"), null);
    table = "t3";
    metadataTable.put(makeExtent(table, "a", null), null);
    metadataTable.put(makeExtent(table, "b", "a"), null);
    metadataTable.put(makeExtent(table, "c", "b"), null);
    metadataTable.put(makeExtent(table, "d", "c"), null);
    metadataTable.put(makeExtent(table, "e", "d"), null);
    metadataTable.put(makeExtent(table, null, "e"), null);
    TestChaoticLoadBalancer balancer = new TestChaoticLoadBalancer();
    SortedMap<TServerInstance, TabletServerStatus> current = new TreeMap<>();
    for (Entry<TServerInstance, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus(entry.getKey()));
    }
    Map<KeyExtent, TServerInstance> assignments = new HashMap<>();
    balancer.getAssignments(getAssignments(servers), metadataTable, assignments);
    assertEquals(assignments.size(), metadataTable.size());
}

Example 10 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class DefaultLoadBalancerTest method checkBalance.

private void checkBalance(List<KeyExtent> metadataTable, Map<TServerInstance, FakeTServer> servers, Map<String, Integer> expectedCounts) {
    // Verify they are spread evenly over the cluster
    int average = metadataTable.size() / servers.size();
    for (FakeTServer server : servers.values()) {
        int diff = server.extents.size() - average;
        if (diff < 0)
            fail("average number of tablets is " + average + " but a server has " + server.extents.size());
        if (diff > 1)
            fail("average number of tablets is " + average + " but a server has " + server.extents.size());
    }
    if (expectedCounts != null) {
        for (FakeTServer server : servers.values()) {
            Map<String, Integer> counts = new HashMap<>();
            for (KeyExtent extent : server.extents) {
                String t = extent.getTableId().canonicalID();
                if (counts.get(t) == null)
                    counts.put(t, 0);
                counts.put(t, counts.get(t) + 1);
            }
            for (Entry<String, Integer> entry : counts.entrySet()) {
                assertEquals(expectedCounts.get(entry.getKey()), counts.get(entry.getKey()));
            }
        }
    }
}