Examples with ClientContext org.apache.accumulo.core.clientImpl.ClientContext used on opensource projects

Example 11 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class ReplicationIT method waitForGCLock.

private void waitForGCLock(AccumuloClient client) throws InterruptedException {
    // Check if the GC process has the lock before wasting our retry attempts
    ZooCache zcache = ((ClientContext) client).getZooCache();
    var zkPath = ServiceLock.path(ZooUtil.getRoot(client.instanceOperations().getInstanceId()) + Constants.ZGC_LOCK);
    log.info("Looking for GC lock at {}", zkPath);
    byte[] data = ServiceLock.getLockData(zcache, zkPath, null);
    while (data == null) {
        log.info("Waiting for GC ZooKeeper lock to be acquired");
        Thread.sleep(1000);
        data = ServiceLock.getLockData(zcache, zkPath, null);
    }
}

Example 12 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class ReplicationOperationsImplIT method getReplicationOperations.

/**
 * Spoof out the Manager so we can call the implementation without starting a full instance.
 */
private ReplicationOperationsImpl getReplicationOperations() {
    Manager manager = EasyMock.createMock(Manager.class);
    EasyMock.expect(manager.getContext()).andReturn(context).anyTimes();
    EasyMock.replay(manager);
    final ManagerClientServiceHandler mcsh = new ManagerClientServiceHandler(manager) {

        @Override
        protected TableId getTableId(ClientContext context, String tableName) {
            try {
                return TableId.of(client.tableOperations().tableIdMap().get(tableName));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    ClientContext context = (ClientContext) client;
    return new ReplicationOperationsImpl(context) {

        @Override
        protected boolean getManagerDrain(final TInfo tinfo, final TCredentials rpcCreds, final String tableName, final Set<String> wals) {
            try {
                return mcsh.drainReplicationTable(tinfo, rpcCreds, tableName, wals);
            } catch (TException e) {
                throw new RuntimeException(e);
            }
        }
    };
}

Example 13 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class InputConfigurator method binOffline.

public static Map<String, Map<KeyExtent, List<Range>>> binOffline(TableId tableId, List<Range> ranges, ClientContext context) throws AccumuloException, TableNotFoundException {
    Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>();
    if (context.getTableState(tableId) != TableState.OFFLINE) {
        context.clearTableListCache();
        if (context.getTableState(tableId) != TableState.OFFLINE) {
            throw new AccumuloException("Table is online tableId:" + tableId + " cannot scan table in offline mode ");
        }
    }
    for (Range range : ranges) {
        Text startRow;
        if (range.getStartKey() != null)
            startRow = range.getStartKey().getRow();
        else
            startRow = new Text();
        Range metadataRange = new Range(new KeyExtent(tableId, startRow, null).toMetaRow(), true, null, false);
        Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
        TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
        scanner.fetchColumnFamily(LastLocationColumnFamily.NAME);
        scanner.fetchColumnFamily(CurrentLocationColumnFamily.NAME);
        scanner.fetchColumnFamily(FutureLocationColumnFamily.NAME);
        scanner.setRange(metadataRange);
        RowIterator rowIter = new RowIterator(scanner);
        KeyExtent lastExtent = null;
        while (rowIter.hasNext()) {
            Iterator<Map.Entry<Key, Value>> row = rowIter.next();
            String last = "";
            KeyExtent extent = null;
            String location = null;
            while (row.hasNext()) {
                Map.Entry<Key, Value> entry = row.next();
                Key key = entry.getKey();
                if (key.getColumnFamily().equals(LastLocationColumnFamily.NAME)) {
                    last = entry.getValue().toString();
                }
                if (key.getColumnFamily().equals(CurrentLocationColumnFamily.NAME) || key.getColumnFamily().equals(FutureLocationColumnFamily.NAME)) {
                    location = entry.getValue().toString();
                }
                if (TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
                    extent = KeyExtent.fromMetaPrevRow(entry);
                }
            }
            if (location != null)
                return null;
            if (!extent.tableId().equals(tableId)) {
                throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
            }
            if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
                throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
            }
            binnedRanges.computeIfAbsent(last, k -> new HashMap<>()).computeIfAbsent(extent, k -> new ArrayList<>()).add(range);
            if (extent.endRow() == null || range.afterEndKey(new Key(extent.endRow()).followingKey(PartialKey.ROW))) {
                break;
            }
            lastExtent = extent;
        }
    }
    return binnedRanges;
}

Example 14 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class TableDiskUsage method getDiskUsage.

public static Map<TreeSet<String>, Long> getDiskUsage(Set<TableId> tableIds, VolumeManager fs, AccumuloClient client) throws IOException {
    TableDiskUsage tdu = new TableDiskUsage();
    // Add each tableID
    for (TableId tableId : tableIds) tdu.addTable(tableId);
    HashSet<TableId> tablesReferenced = new HashSet<>(tableIds);
    HashSet<TableId> emptyTableIds = new HashSet<>();
    HashSet<String> nameSpacesReferenced = new HashSet<>();
    // For each table ID
    for (TableId tableId : tableIds) {
        Scanner mdScanner;
        try {
            mdScanner = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
        } catch (TableNotFoundException e) {
            throw new RuntimeException(e);
        }
        mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
        mdScanner.setRange(new KeyExtent(tableId, null, null).toMetaRange());
        if (!mdScanner.iterator().hasNext()) {
            emptyTableIds.add(tableId);
        }
        // Read each file referenced by that table
        for (Entry<Key, Value> entry : mdScanner) {
            String file = entry.getKey().getColumnQualifier().toString();
            String[] parts = file.split("/");
            // the filename
            String uniqueName = parts[parts.length - 1];
            if (file.contains(":") || file.startsWith("../")) {
                String ref = parts[parts.length - 3];
                // Track any tables which are referenced externally by the current table
                if (!ref.equals(tableId.canonical())) {
                    tablesReferenced.add(TableId.of(ref));
                }
                if (file.contains(":") && parts.length > 3) {
                    List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3));
                    nameSpacesReferenced.add(Joiner.on("/").join(base));
                }
            }
            // add this file to this table
            tdu.linkFileAndTable(tableId, uniqueName);
        }
    }
    // Each table seen (provided by user, or reference by table the user provided)
    for (TableId tableId : tablesReferenced) {
        for (String tableDir : nameSpacesReferenced) {
            // Find each file and add its size
            Path path = new Path(tableDir + "/" + tableId);
            if (!fs.exists(path)) {
                log.debug("Table ID directory {} does not exist.", path);
                continue;
            }
            log.info("Get all files recursively in {}", path);
            RemoteIterator<LocatedFileStatus> ri = fs.listFiles(path, true);
            while (ri.hasNext()) {
                FileStatus status = ri.next();
                String name = status.getPath().getName();
                tdu.addFileSize(name, status.getLen());
            }
        }
    }
    Map<TableId, String> reverseTableIdMap = ((ClientContext) client).getTableIdToNameMap();
    TreeMap<TreeSet<String>, Long> usage = new TreeMap<>((o1, o2) -> {
        int len1 = o1.size();
        int len2 = o2.size();
        int min = Math.min(len1, len2);
        Iterator<String> iter1 = o1.iterator();
        Iterator<String> iter2 = o2.iterator();
        int count = 0;
        while (count < min) {
            String s1 = iter1.next();
            String s2 = iter2.next();
            int cmp = s1.compareTo(s2);
            if (cmp != 0)
                return cmp;
            count++;
        }
        return len1 - len2;
    });
    for (Entry<List<TableId>, Long> entry : tdu.calculateUsage().entrySet()) {
        TreeSet<String> tableNames = new TreeSet<>();
        // Convert size shared by each table id into size shared by each table name
        for (TableId tableId : entry.getKey()) tableNames.add(reverseTableIdMap.get(tableId));
        // Make table names to shared file size
        usage.put(tableNames, entry.getValue());
    }
    if (!emptyTableIds.isEmpty()) {
        TreeSet<String> emptyTables = new TreeSet<>();
        for (TableId tableId : emptyTableIds) {
            emptyTables.add(reverseTableIdMap.get(tableId));
        }
        usage.put(emptyTables, 0L);
    }
    return usage;
}

Example 15 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class SuspendedTabletsIT method setUp.

@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        // Wait for all tablet servers to come online and then choose the first server in the list.
        // Update the balancer configuration to assign all metadata tablets to that server (and
        // everything else to other servers).
        InstanceOperations iops = client.instanceOperations();
        List<String> tservers = iops.getTabletServers();
        while (tservers == null || tservers.size() < 1) {
            Thread.sleep(1000L);
            tservers = client.instanceOperations().getTabletServers();
        }
        HostAndPort metadataServer = HostAndPort.fromString(tservers.get(0));
        log.info("Configuring balancer to assign all metadata tablets to {}", metadataServer);
        iops.setProperty(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + MetadataTable.NAME, metadataServer.toString());
        // Wait for the balancer to assign all metadata tablets to the chosen server.
        ClientContext ctx = (ClientContext) client;
        TabletLocations tl = TabletLocations.retrieve(ctx, MetadataTable.NAME, RootTable.NAME);
        while (tl.hosted.keySet().size() != 1 || !tl.hosted.containsKey(metadataServer)) {
            log.info("Metadata tablets are not hosted on the correct server. Waiting for balancer...");
            Thread.sleep(1000L);
            tl = TabletLocations.retrieve(ctx, MetadataTable.NAME, RootTable.NAME);
        }
        log.info("Metadata tablets are now hosted on {}", metadataServer);
    }
    // Since we started only a single tablet server, we know it's the one hosting the
    // metadata table. Save its process reference off so we can exclude it later when
    // killing tablet servers.
    Collection<ProcessReference> procs = getCluster().getProcesses().get(ServerType.TABLET_SERVER);
    assertEquals("Expected a single tserver process", 1, procs.size());
    metadataTserverProcess = procs.iterator().next();
    // Update the number of tservers and start the new tservers.
    getCluster().getConfig().setNumTservers(TSERVERS);
    getCluster().start();
}