Examples with TableDescriptor org.apache.hadoop.hbase.client.TableDescriptor used on opensource projects

Example 46 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class ExpiredMobFileCleaner method run.

@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "REC_CATCH_EXCEPTION", justification = "Intentional")
@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        printUsage();
        return 1;
    }
    String tableName = args[0];
    String familyName = args[1];
    TableName tn = TableName.valueOf(tableName);
    Connection connection = ConnectionFactory.createConnection(getConf());
    Admin admin = connection.getAdmin();
    try {
        TableDescriptor htd = admin.getDescriptor(tn);
        ColumnFamilyDescriptor family = htd.getColumnFamily(Bytes.toBytes(familyName));
        if (family == null || !family.isMobEnabled()) {
            throw new IOException("Column family " + familyName + " is not a MOB column family");
        }
        if (family.getMinVersions() > 0) {
            throw new IOException("The minVersions of the column family is not 0, could not be handled by this cleaner");
        }
        cleanExpiredMobFiles(tableName, family);
        return 0;
    } finally {
        admin.close();
        try {
            connection.close();
        } catch (IOException e) {
            LOG.error("Failed to close the connection.", e);
        }
    }
}

Example 47 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class UpdatePeerConfigProcedure method updateLastPushedSequenceIdForSerialPeer.

@Override
protected void updateLastPushedSequenceIdForSerialPeer(MasterProcedureEnv env) throws IOException, ReplicationException {
    if (!oldPeerConfig.isSerial()) {
        assert peerConfig.isSerial();
        // change to serial
        setLastPushedSequenceId(env, peerConfig);
        return;
    }
    if (!peerConfig.isSerial()) {
        // remove the serial flag
        env.getReplicationPeerManager().removeAllLastPushedSeqIds(peerId);
        return;
    }
    // enter here means peerConfig and oldPeerConfig are both serial, let's find out the diffs and
    // process them
    ReplicationQueueStorage queueStorage = env.getReplicationPeerManager().getQueueStorage();
    Connection conn = env.getMasterServices().getConnection();
    Map<String, Long> lastSeqIds = new HashMap<String, Long>();
    List<String> encodedRegionNames = new ArrayList<>();
    for (TableDescriptor td : env.getMasterServices().getTableDescriptors().getAll().values()) {
        if (!td.hasGlobalReplicationScope()) {
            continue;
        }
        TableName tn = td.getTableName();
        if (oldPeerConfig.needToReplicate(tn)) {
            if (!peerConfig.needToReplicate(tn)) {
                // removed from peer config
                for (String encodedRegionName : ReplicationBarrierFamilyFormat.getTableEncodedRegionNamesForSerialReplication(conn, tn)) {
                    addToList(encodedRegionNames, encodedRegionName, queueStorage);
                }
            }
        } else if (peerConfig.needToReplicate(tn)) {
            // newly added to peer config
            setLastPushedSequenceIdForTable(env, tn, lastSeqIds);
        }
    }
    if (!encodedRegionNames.isEmpty()) {
        queueStorage.removeLastSequenceIds(peerId, encodedRegionNames);
    }
    if (!lastSeqIds.isEmpty()) {
        queueStorage.setLastSequenceIds(peerId, lastSeqIds);
    }
}

Example 48 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class MobFileCleanerChore method chore.

@Override
protected void chore() {
    TableDescriptors htds = master.getTableDescriptors();
    Map<String, TableDescriptor> map = null;
    try {
        map = htds.getAll();
    } catch (IOException e) {
        LOG.error("MobFileCleanerChore failed", e);
        return;
    }
    for (TableDescriptor htd : map.values()) {
        for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
            if (hcd.isMobEnabled() && hcd.getMinVersions() == 0) {
                try {
                    cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd);
                } catch (IOException e) {
                    LOG.error("Failed to clean the expired mob files table={} family={}", htd.getTableName().getNameAsString(), hcd.getNameAsString(), e);
                }
            }
        }
        try {
            // Now clean obsolete files for a table
            LOG.info("Cleaning obsolete MOB files from table={}", htd.getTableName());
            cleanupObsoleteMobFiles(master.getConfiguration(), htd.getTableName());
            LOG.info("Cleaning obsolete MOB files finished for table={}", htd.getTableName());
        } catch (IOException e) {
            LOG.error("Failed to clean the obsolete mob files for table={}", htd.getTableName(), e);
        }
    }
}

Example 49 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class MobFileCleanerChore method cleanupObsoleteMobFiles.

/**
 * Performs housekeeping file cleaning (called by MOB Cleaner chore)
 * @param conf configuration
 * @param table table name
 * @throws IOException exception
 */
public void cleanupObsoleteMobFiles(Configuration conf, TableName table) throws IOException {
    long minAgeToArchive = conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE);
    // We check only those MOB files, which creation time is less
    // than maxCreationTimeToArchive. This is a current time - 1h. 1 hour gap
    // gives us full confidence that all corresponding store files will
    // exist at the time cleaning procedure begins and will be examined.
    // So, if MOB file creation time is greater than this maxTimeToArchive,
    // this will be skipped and won't be archived.
    long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive;
    try (final Connection conn = ConnectionFactory.createConnection(conf);
        final Admin admin = conn.getAdmin()) {
        TableDescriptor htd = admin.getDescriptor(table);
        List<ColumnFamilyDescriptor> list = MobUtils.getMobColumnFamilies(htd);
        if (list.size() == 0) {
            LOG.info("Skipping non-MOB table [{}]", table);
            return;
        } else {
            LOG.info("Only MOB files whose creation time older than {} will be archived, table={}", maxCreationTimeToArchive, table);
        }
        Path rootDir = CommonFSUtils.getRootDir(conf);
        Path tableDir = CommonFSUtils.getTableDir(rootDir, table);
        // How safe is this call?
        List<Path> regionDirs = FSUtils.getRegionDirs(FileSystem.get(conf), tableDir);
        Set<String> allActiveMobFileName = new HashSet<String>();
        FileSystem fs = FileSystem.get(conf);
        for (Path regionPath : regionDirs) {
            for (ColumnFamilyDescriptor hcd : list) {
                String family = hcd.getNameAsString();
                Path storePath = new Path(regionPath, family);
                boolean succeed = false;
                Set<String> regionMobs = new HashSet<String>();
                while (!succeed) {
                    if (!fs.exists(storePath)) {
                        String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore" + " execution, aborting MOB file cleaner chore.", storePath);
                        throw new IOException(errMsg);
                    }
                    RemoteIterator<LocatedFileStatus> rit = fs.listLocatedStatus(storePath);
                    List<Path> storeFiles = new ArrayList<Path>();
                    // Load list of store files first
                    while (rit.hasNext()) {
                        Path p = rit.next().getPath();
                        if (fs.isFile(p)) {
                            storeFiles.add(p);
                        }
                    }
                    LOG.info("Found {} store files in: {}", storeFiles.size(), storePath);
                    Path currentPath = null;
                    try {
                        for (Path pp : storeFiles) {
                            currentPath = pp;
                            LOG.trace("Store file: {}", pp);
                            HStoreFile sf = new HStoreFile(fs, pp, conf, CacheConfig.DISABLED, BloomType.NONE, true);
                            sf.initReader();
                            byte[] mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);
                            byte[] bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY);
                            // close store file to avoid memory leaks
                            sf.closeStoreFile(true);
                            if (mobRefData == null) {
                                if (bulkloadMarkerData == null) {
                                    LOG.warn("Found old store file with no MOB_FILE_REFS: {} - " + "can not proceed until all old files will be MOB-compacted.", pp);
                                    return;
                                } else {
                                    LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp);
                                    continue;
                                }
                            }
                            // mob compaction code.
                            try {
                                SetMultimap<TableName, String> mobs = MobUtils.deserializeMobFileRefs(mobRefData).build();
                                LOG.debug("Found {} mob references for store={}", mobs.size(), sf);
                                LOG.trace("Specific mob references found for store={} : {}", sf, mobs);
                                regionMobs.addAll(mobs.values());
                            } catch (RuntimeException exception) {
                                throw new IOException("failure getting mob references for hfile " + sf, exception);
                            }
                        }
                    } catch (FileNotFoundException e) {
                        LOG.warn("Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error", currentPath, e);
                        regionMobs.clear();
                        continue;
                    }
                    succeed = true;
                }
                // Add MOB references for current region/family
                allActiveMobFileName.addAll(regionMobs);
            }
        // END column families
        }
        // Check if number of MOB files too big (over 1M)
        if (allActiveMobFileName.size() > 1000000) {
            LOG.warn("Found too many active MOB files: {}, table={}, " + "this may result in high memory pressure.", allActiveMobFileName.size(), table);
        }
        LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table);
        allActiveMobFileName.stream().forEach(LOG::trace);
        // Now scan MOB directories and find MOB files with no references to them
        for (ColumnFamilyDescriptor hcd : list) {
            List<Path> toArchive = new ArrayList<Path>();
            String family = hcd.getNameAsString();
            Path dir = MobUtils.getMobFamilyPath(conf, table, family);
            RemoteIterator<LocatedFileStatus> rit = fs.listLocatedStatus(dir);
            while (rit.hasNext()) {
                LocatedFileStatus lfs = rit.next();
                Path p = lfs.getPath();
                if (!allActiveMobFileName.contains(p.getName())) {
                    // MOB is not in a list of active references, but it can be too
                    // fresh, skip it in this case
                    long creationTime = fs.getFileStatus(p).getModificationTime();
                    if (creationTime < maxCreationTimeToArchive) {
                        LOG.trace("Archiving MOB file {} creation time={}", p, (fs.getFileStatus(p).getModificationTime()));
                        toArchive.add(p);
                    } else {
                        LOG.trace("Skipping fresh file: {}. Creation time={}", p, fs.getFileStatus(p).getModificationTime());
                    }
                } else {
                    LOG.trace("Keeping active MOB file: {}", p);
                }
            }
            LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(), table, family);
            archiveMobFiles(conf, table, family.getBytes(), toArchive);
            LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, family);
        }
    }
}

Example 50 with TableDescriptor

use of org.apache.hadoop.hbase.client.TableDescriptor in project hbase by apache.

the class BackupManager method createBackupInfo.

/**
 * Creates a backup info based on input backup request.
 * @param backupId backup id
 * @param type type
 * @param tableList table list
 * @param targetRootDir root dir
 * @param workers number of parallel workers
 * @param bandwidth bandwidth per worker in MB per sec
 * @return BackupInfo
 * @throws BackupException exception
 */
public BackupInfo createBackupInfo(String backupId, BackupType type, List<TableName> tableList, String targetRootDir, int workers, long bandwidth) throws BackupException {
    if (targetRootDir == null) {
        throw new BackupException("Wrong backup request parameter: target backup root directory");
    }
    if (type == BackupType.FULL && (tableList == null || tableList.isEmpty())) {
        // If table list is null for full backup, which means backup all tables. Then fill the table
        // list with all user tables from meta. It no table available, throw the request exception.
        List<TableDescriptor> htds = null;
        try (Admin admin = conn.getAdmin()) {
            htds = admin.listTableDescriptors();
        } catch (Exception e) {
            throw new BackupException(e);
        }
        if (htds == null) {
            throw new BackupException("No table exists for full backup of all tables.");
        } else {
            tableList = new ArrayList<>();
            for (TableDescriptor hTableDescriptor : htds) {
                TableName tn = hTableDescriptor.getTableName();
                if (tn.equals(BackupSystemTable.getTableName(conf))) {
                    // skip backup system table
                    continue;
                }
                tableList.add(hTableDescriptor.getTableName());
            }
            LOG.info("Full backup all the tables available in the cluster: {}", tableList);
        }
    }
    // there are one or more tables in the table list
    backupInfo = new BackupInfo(backupId, type, tableList.toArray(new TableName[tableList.size()]), targetRootDir);
    backupInfo.setBandwidth(bandwidth);
    backupInfo.setWorkers(workers);
    return backupInfo;
}