Examples with DrRegionList io.datarouter.client.hbase.cluster.DrRegionListFactory.DrRegionList used on opensource projects

Example 1 with DrRegionList

use of io.datarouter.client.hbase.cluster.DrRegionListFactory.DrRegionList in project datarouter by hotpads.

the class HBaseRegionBalancerJob method balanceClient.

// return true if completed balancing
private boolean balanceClient(TaskTracker tracker, ClientId clientId) {
    logger.warn("calculating region movements for client {}", clientId.getName());
    Admin admin = hBaseClientManager.getAdmin(clientId);
    List<HBaseRegionMovement> movements = new ArrayList<>();
    DrServerList serverList = new DrServerList(admin);
    List<String> tableNames = nodes.getTableNamesForClient(clientId.getName());
    Collections.sort(tableNames);
    int tableCounter = 0;
    for (String tableName : tableNames) {
        if (tracker.shouldStop()) {
            return false;
        }
        ++tableCounter;
        PhaseTimer timer = new PhaseTimer("generating movements for table " + tableName + " #" + tableCounter + "/" + tableNames.size());
        List<HBaseRegionMovement> tableMovements = new ArrayList<>();
        PhysicalNode<?, ?, ?> physicalNodeForTable = nodes.getPhysicalNodeForClientAndTable(clientId.getName(), tableName);
        BaseHBaseRegionBalancer balancer = balancerFactory.getBalancerForTable(clientId, tableName);
        DrRegionList regionList;
        try {
            regionList = drRegionListFactory.make(clientId, serverList, tableName, physicalNodeForTable, balancer);
        } catch (Exception e) {
            logger.error("skipping table " + tableName, e);
            continue;
        }
        for (DrRegionInfo<?> region : regionList.getRegions()) {
            if (region.isNotOnAnyServer()) {
                logger.warn("region {} is not currently hosted, so not attempting to move it", region.getRegion().getRegionNameAsString());
                continue;
            }
            if (region.isOnCorrectServer()) {
                continue;
            }
            HBaseRegionMovement movement = new HBaseRegionMovement(tableName, region.getRegion().getEncodedName(), region.getHBaseServerName(), region.getBalancerDestinationHBaseServerName());
            tableMovements.add(movement);
        }
        timer.add("generated " + tableMovements.size() + " movements");
        logger.warn(timer.toString());
        movements.addAll(tableMovements);
    }
    // we calculated all the movements.  now move them
    long iterationStartTimeMs = System.currentTimeMillis();
    Map<ServerName, List<HBaseRegionMovement>> movementsByCurrentServer = HBaseRegionMovement.getByCurrentServer(movements);
    int clusterMovementCounter = 0;
    logger.warn("processing {} total movements", movements.size());
    for (DrServerInfo serverInfo : serverList.getServersSortedByDescendingLoad()) {
        List<HBaseRegionMovement> movementsForServer = movementsByCurrentServer.getOrDefault(serverInfo.getServerName(), List.of());
        logger.warn("expecting {} movements for server {}", movementsForServer.size(), serverInfo.getServerName());
    }
    for (DrServerInfo serverInfo : serverList.getServersSortedByDescendingLoad()) {
        List<HBaseRegionMovement> movementsForServer = movementsByCurrentServer.getOrDefault(serverInfo.getServerName(), List.of());
        logger.warn("processing {} movements for server {}", movementsForServer.size(), serverInfo.getServerName());
        int serverMovementCounter = 0;
        for (HBaseRegionMovement movement : movementsForServer) {
            ++clusterMovementCounter;
            ++serverMovementCounter;
            logger.warn("moving region {}/{} ({}/{}): {}", serverMovementCounter, movementsForServer.size(), clusterMovementCounter, movements.size(), movement);
            try {
                admin.move(movement.getRegionNameBytes(), movement.getDestinationServerNameBytes());
            } catch (Exception ex) {
                // in 0.94, this is UndeclaredThrowableException wrapping other Exceptions
                logger.error("exception moving region, skipping", ex);
            }
            ThreadTool.sleepUnchecked(hbaseSettings.getSleepBetweenRegionMovementMs());
            if (tracker.increment().shouldStop()) {
                return false;
            }
            if (System.currentTimeMillis() - iterationStartTimeMs > RECALCULATE_AFTER_MS) {
                logger.warn("suspending to check for new servers", RECALCULATE_AFTER_MS);
                return false;
            }
        }
        logger.warn("processed {} movements for server {}", serverMovementCounter, serverInfo.getServerName());
    }
    return true;
}

Example 2 with DrRegionList

use of io.datarouter.client.hbase.cluster.DrRegionListFactory.DrRegionList in project datarouter by hotpads.

the class HBaseCompactionJob method compactTable.

private void compactTable(TaskTracker tracker, Admin admin, DrServerList servers, ClientId clientId, String tableName) {
    numTables.increment();
    PhysicalNode<?, ?, ?> physicalNodeForTable = nodes.getPhysicalNodeForClientAndTable(clientId.getName(), tableName);
    if (physicalNodeForTable == null) {
        logger.error("physicalNode not found for table " + tableName);
        return;
    }
    DrRegionList regionList = drRegionListFactory.make(clientId, servers, tableName, physicalNodeForTable, balancerFactory.getBalancerForTable(clientId, tableName));
    LongAdder numRegionsTriggeredInTable = new LongAdder();
    for (DrRegionInfo<?> region : regionList.getRegions()) {
        if (tracker.heartbeat(numRegions.sum()).shouldStop()) {
            return;
        }
        numRegions.increment();
        if (region.getLoad() == null) {
            numMissingLoad.increment();
            logger.warn("region.getLoad()==null on {} {}", region.getTableName(), region.getName());
            continue;
        }
        if (region.getLoad().getDataLocality() == 1F) {
            numAlreadyLocal.increment();
            continue;
        }
        DrhCompactionScheduler<?> scheduler = new DrhCompactionScheduler<>(compactionInfo, region);
        if (scheduler.shouldCompact()) {
            compactRegion(admin, region, numRegionsTriggeredInTable);
        }
    }
}