Examples with VisibleForTesting com.google.common.annotations.VisibleForTesting used on opensource projects

Example 6 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class XDR method getBytes.

@VisibleForTesting
public byte[] getBytes() {
    ByteBuffer d = asReadOnlyWrap().buffer();
    byte[] b = new byte[d.remaining()];
    d.get(b);
    return b;
}

Example 7 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class NNStorage method setStorageDirectories.

/**
   * Set the storage directories which will be used. This should only ever be
   * called from inside NNStorage. However, it needs to remain package private
   * for testing, as StorageDirectories need to be reinitialised after using
   * Mockito.spy() on this class, as Mockito doesn't work well with inner
   * classes, such as StorageDirectory in this case.
   *
   * Synchronized due to initialization of storageDirs and removedStorageDirs.
   *
   * @param fsNameDirs Locations to store images.
   * @param fsEditsDirs Locations to store edit logs.
   * @throws IOException
   */
@VisibleForTesting
synchronized void setStorageDirectories(Collection<URI> fsNameDirs, Collection<URI> fsEditsDirs, Collection<URI> sharedEditsDirs) throws IOException {
    getStorageDirs().clear();
    this.removedStorageDirs.clear();
    // Add all name dirs with appropriate NameNodeDirType
    for (URI dirName : fsNameDirs) {
        checkSchemeConsistency(dirName);
        boolean isAlsoEdits = false;
        for (URI editsDirName : fsEditsDirs) {
            if (editsDirName.compareTo(dirName) == 0) {
                isAlsoEdits = true;
                fsEditsDirs.remove(editsDirName);
                break;
            }
        }
        NameNodeDirType dirType = (isAlsoEdits) ? NameNodeDirType.IMAGE_AND_EDITS : NameNodeDirType.IMAGE;
        // URI is of type file://
        if (dirName.getScheme().compareTo("file") == 0) {
            this.addStorageDir(new StorageDirectory(new File(dirName.getPath()), dirType, // Don't lock the dir if it's shared.
            sharedEditsDirs.contains(dirName)));
        }
    }
    // Add edits dirs if they are different from name dirs
    for (URI dirName : fsEditsDirs) {
        checkSchemeConsistency(dirName);
        // URI is of type file://
        if (dirName.getScheme().compareTo("file") == 0) {
            this.addStorageDir(new StorageDirectory(new File(dirName.getPath()), NameNodeDirType.EDITS, sharedEditsDirs.contains(dirName)));
        }
    }
}

Example 8 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class NameNode method parseArguments.

@VisibleForTesting
static StartupOption parseArguments(String[] args) {
    int argsLen = (args == null) ? 0 : args.length;
    StartupOption startOpt = StartupOption.REGULAR;
    for (int i = 0; i < argsLen; i++) {
        String cmd = args[i];
        if (StartupOption.FORMAT.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.FORMAT;
            for (i = i + 1; i < argsLen; i++) {
                if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
                    i++;
                    if (i >= argsLen) {
                        // if no cluster id specified, return null
                        LOG.error("Must specify a valid cluster ID after the " + StartupOption.CLUSTERID.getName() + " flag");
                        return null;
                    }
                    String clusterId = args[i];
                    // Make sure an id is specified and not another flag
                    if (clusterId.isEmpty() || clusterId.equalsIgnoreCase(StartupOption.FORCE.getName()) || clusterId.equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) {
                        LOG.error("Must specify a valid cluster ID after the " + StartupOption.CLUSTERID.getName() + " flag");
                        return null;
                    }
                    startOpt.setClusterId(clusterId);
                }
                if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) {
                    startOpt.setForceFormat(true);
                }
                if (args[i].equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) {
                    startOpt.setInteractiveFormat(false);
                }
            }
        } else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.GENCLUSTERID;
        } else if (StartupOption.REGULAR.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.REGULAR;
        } else if (StartupOption.BACKUP.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.BACKUP;
        } else if (StartupOption.CHECKPOINT.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.CHECKPOINT;
        } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd) || StartupOption.UPGRADEONLY.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd) ? StartupOption.UPGRADE : StartupOption.UPGRADEONLY;
            /* Can be followed by CLUSTERID with a required parameter or
         * RENAMERESERVED with an optional parameter
         */
            while (i + 1 < argsLen) {
                String flag = args[i + 1];
                if (flag.equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
                    if (i + 2 < argsLen) {
                        i += 2;
                        startOpt.setClusterId(args[i]);
                    } else {
                        LOG.error("Must specify a valid cluster ID after the " + StartupOption.CLUSTERID.getName() + " flag");
                        return null;
                    }
                } else if (flag.equalsIgnoreCase(StartupOption.RENAMERESERVED.getName())) {
                    if (i + 2 < argsLen) {
                        FSImageFormat.setRenameReservedPairs(args[i + 2]);
                        i += 2;
                    } else {
                        FSImageFormat.useDefaultRenameReservedPairs();
                        i += 1;
                    }
                } else {
                    LOG.error("Unknown upgrade flag " + flag);
                    return null;
                }
            }
        } else if (StartupOption.ROLLINGUPGRADE.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.ROLLINGUPGRADE;
            ++i;
            if (i >= argsLen) {
                LOG.error("Must specify a rolling upgrade startup option " + RollingUpgradeStartupOption.getAllOptionString());
                return null;
            }
            startOpt.setRollingUpgradeStartupOption(args[i]);
        } else if (StartupOption.ROLLBACK.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.ROLLBACK;
        } else if (StartupOption.IMPORT.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.IMPORT;
        } else if (StartupOption.BOOTSTRAPSTANDBY.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.BOOTSTRAPSTANDBY;
            return startOpt;
        } else if (StartupOption.INITIALIZESHAREDEDITS.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.INITIALIZESHAREDEDITS;
            for (i = i + 1; i < argsLen; i++) {
                if (StartupOption.NONINTERACTIVE.getName().equals(args[i])) {
                    startOpt.setInteractiveFormat(false);
                } else if (StartupOption.FORCE.getName().equals(args[i])) {
                    startOpt.setForceFormat(true);
                } else {
                    LOG.error("Invalid argument: " + args[i]);
                    return null;
                }
            }
            return startOpt;
        } else if (StartupOption.RECOVER.getName().equalsIgnoreCase(cmd)) {
            if (startOpt != StartupOption.REGULAR) {
                throw new RuntimeException("Can't combine -recover with " + "other startup options.");
            }
            startOpt = StartupOption.RECOVER;
            while (++i < argsLen) {
                if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) {
                    startOpt.setForce(MetaRecoveryContext.FORCE_FIRST_CHOICE);
                } else {
                    throw new RuntimeException("Error parsing recovery options: " + "can't understand option \"" + args[i] + "\"");
                }
            }
        } else if (StartupOption.METADATAVERSION.getName().equalsIgnoreCase(cmd)) {
            startOpt = StartupOption.METADATAVERSION;
        } else {
            return null;
        }
    }
    return startOpt;
}

Example 9 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class KMSACLs method setKeyACLs.

@VisibleForTesting
void setKeyACLs(Configuration conf) {
    Map<String, HashMap<KeyOpType, AccessControlList>> tempKeyAcls = new HashMap<String, HashMap<KeyOpType, AccessControlList>>();
    Map<String, String> allKeyACLS = conf.getValByRegex(KMSConfiguration.KEY_ACL_PREFIX_REGEX);
    for (Map.Entry<String, String> keyAcl : allKeyACLS.entrySet()) {
        String k = keyAcl.getKey();
        // this should be of type "key.acl.<KEY_NAME>.<OP_TYPE>"
        int keyNameStarts = KMSConfiguration.KEY_ACL_PREFIX.length();
        int keyNameEnds = k.lastIndexOf(".");
        if (keyNameStarts >= keyNameEnds) {
            LOG.warn("Invalid key name '{}'", k);
        } else {
            String aclStr = keyAcl.getValue();
            String keyName = k.substring(keyNameStarts, keyNameEnds);
            String keyOp = k.substring(keyNameEnds + 1);
            KeyOpType aclType = null;
            try {
                aclType = KeyOpType.valueOf(keyOp);
            } catch (IllegalArgumentException e) {
                LOG.warn("Invalid key Operation '{}'", keyOp);
            }
            if (aclType != null) {
                // On the assumption this will be single threaded.. else we need to
                // ConcurrentHashMap
                HashMap<KeyOpType, AccessControlList> aclMap = tempKeyAcls.get(keyName);
                if (aclMap == null) {
                    aclMap = new HashMap<KeyOpType, AccessControlList>();
                    tempKeyAcls.put(keyName, aclMap);
                }
                aclMap.put(aclType, new AccessControlList(aclStr));
                LOG.info("KEY_NAME '{}' KEY_OP '{}' ACL '{}'", keyName, aclType, aclStr);
            }
        }
    }
    keyAcls = tempKeyAcls;
    final Map<KeyOpType, AccessControlList> tempDefaults = new HashMap<>();
    final Map<KeyOpType, AccessControlList> tempWhitelists = new HashMap<>();
    for (KeyOpType keyOp : KeyOpType.values()) {
        parseAclsWithPrefix(conf, KMSConfiguration.DEFAULT_KEY_ACL_PREFIX, keyOp, tempDefaults);
        parseAclsWithPrefix(conf, KMSConfiguration.WHITELIST_KEY_ACL_PREFIX, keyOp, tempWhitelists);
    }
    defaultKeyAcls = tempDefaults;
    whitelistKeyAcls = tempWhitelists;
}

Example 10 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class RMCommunicator method doUnregistration.

@VisibleForTesting
protected void doUnregistration() throws YarnException, IOException, InterruptedException {
    FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
    JobImpl jobImpl = (JobImpl) job;
    if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
        finishState = FinalApplicationStatus.SUCCEEDED;
    } else if (jobImpl.getInternalState() == JobStateInternal.KILLED || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
        finishState = FinalApplicationStatus.KILLED;
    } else if (jobImpl.getInternalState() == JobStateInternal.FAILED || jobImpl.getInternalState() == JobStateInternal.ERROR) {
        finishState = FinalApplicationStatus.FAILED;
    }
    StringBuffer sb = new StringBuffer();
    for (String s : job.getDiagnostics()) {
        sb.append(s).append("\n");
    }
    LOG.info("Setting job diagnostics to " + sb.toString());
    String historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(), context.getApplicationID());
    LOG.info("History url is " + historyUrl);
    FinishApplicationMasterRequest request = FinishApplicationMasterRequest.newInstance(finishState, sb.toString(), historyUrl);
    try {
        while (true) {
            FinishApplicationMasterResponse response = scheduler.finishApplicationMaster(request);
            if (response.getIsUnregistered()) {
                // When excepting ClientService, other services are already stopped,
                // it is safe to let clients know the final states. ClientService
                // should wait for some time so clients have enough time to know the
                // final states.
                RunningAppContext raContext = (RunningAppContext) context;
                raContext.markSuccessfulUnregistration();
                break;
            }
            LOG.info("Waiting for application to be successfully unregistered.");
            Thread.sleep(rmPollInterval);
        }
    } catch (ApplicationMasterNotRegisteredException e) {
        // RM might have restarted or failed over and so lost the fact that AM had
        // registered before.
        register();
        doUnregistration();
    }
}