Examples with OperationType org.apache.hadoop.fs.slive.Constants.OperationType used on opensource projects

Example 6 with OperationType

use of org.apache.hadoop.fs.slive.Constants.OperationType in project hadoop by apache.

the class ConfigMerger method handleOperations.

/**
   * Handles the specific task of merging operations from the command line or
   * extractor object into the base configuration provided
   * 
   * @param opts
   *          the parsed command line option output
   * @param base
   *          the base configuration to merge with
   * @param extractor
   *          the access object to fetch operations from if none from the
   *          command line
   * @return merged configuration object
   * @throws ConfigException
   *           when verification fails
   */
private Configuration handleOperations(ParsedOutput opts, Configuration base, ConfigExtractor extractor) throws ConfigException {
    // get the base set to start off with
    Map<OperationType, OperationData> operations = getBaseOperations();
    // merge with what is coming from config
    Map<OperationType, OperationData> cfgOperations = extractor.getOperations();
    for (OperationType opType : cfgOperations.keySet()) {
        operations.put(opType, cfgOperations.get(opType));
    }
    // see if any coming in from the command line
    for (OperationType opType : OperationType.values()) {
        String opName = opType.lowerName();
        String opVal = opts.getValue(opName);
        if (opVal != null) {
            operations.put(opType, new OperationData(opVal));
        }
    }
    // remove those with <= zero percent
    {
        Map<OperationType, OperationData> cleanedOps = new HashMap<OperationType, OperationData>();
        for (OperationType opType : operations.keySet()) {
            OperationData data = operations.get(opType);
            if (data.getPercent() == null || data.getPercent() > 0.0d) {
                cleanedOps.put(opType, data);
            }
        }
        operations = cleanedOps;
    }
    if (operations.isEmpty()) {
        throw new ConfigException("No operations provided!");
    }
    // verify and adjust
    double currPct = 0;
    int needFill = 0;
    for (OperationType type : operations.keySet()) {
        OperationData op = operations.get(type);
        if (op.getPercent() != null) {
            currPct += op.getPercent();
        } else {
            needFill++;
        }
    }
    if (currPct > 1) {
        throw new ConfigException("Unable to have accumlative percent greater than 100%");
    }
    if (needFill > 0 && currPct < 1) {
        double leftOver = 1.0 - currPct;
        Map<OperationType, OperationData> mpcp = new HashMap<OperationType, OperationData>();
        for (OperationType type : operations.keySet()) {
            OperationData op = operations.get(type);
            if (op.getPercent() == null) {
                op = new OperationData(op.getDistribution(), (leftOver / needFill));
            }
            mpcp.put(type, op);
        }
        operations = mpcp;
    } else if (needFill == 0 && currPct < 1) {
        // redistribute
        double leftOver = 1.0 - currPct;
        Map<OperationType, OperationData> mpcp = new HashMap<OperationType, OperationData>();
        double each = leftOver / operations.keySet().size();
        for (OperationType t : operations.keySet()) {
            OperationData op = operations.get(t);
            op = new OperationData(op.getDistribution(), (op.getPercent() + each));
            mpcp.put(t, op);
        }
        operations = mpcp;
    } else if (needFill > 0 && currPct >= 1) {
        throw new ConfigException(needFill + " unfilled operations but no percentage left to fill with");
    }
    // save into base
    for (OperationType opType : operations.keySet()) {
        String opName = opType.lowerName();
        OperationData opData = operations.get(opType);
        String distr = opData.getDistribution().lowerName();
        String ratio = new Double(opData.getPercent() * 100.0d).toString();
        base.set(String.format(Constants.OP, opName), opData.toString());
        base.set(String.format(Constants.OP_DISTR, opName), distr);
        base.set(String.format(Constants.OP_PERCENT, opName), ratio);
    }
    return base;
}

Example 7 with OperationType

use of org.apache.hadoop.fs.slive.Constants.OperationType in project hadoop by apache.

the class ConfigExtractor method getOperations.

/**
   * @return the map of operations to perform using config (percent may be null
   *         if unspecified)
   */
Map<OperationType, OperationData> getOperations() {
    Map<OperationType, OperationData> operations = new HashMap<OperationType, OperationData>();
    for (OperationType type : OperationType.values()) {
        String opname = type.lowerName();
        String keyname = String.format(Constants.OP, opname);
        String kval = config.get(keyname);
        if (kval == null) {
            continue;
        }
        operations.put(type, new OperationData(kval));
    }
    return operations;
}