Search in sources :

Example 11 with JsonIgnore

use of com.fasterxml.jackson.annotation.JsonIgnore in project ambrose by twitter.

the class PigJob method setJobStats.

@JsonIgnore
public void setJobStats(JobStats stats) {
    this.inputInfoList = inputInfoList(stats.getInputs());
    this.outputInfoList = outputInfoList(stats.getOutputs());
    // job metrics
    Map<String, Number> metrics = Maps.newHashMap();
    metrics.put("hdfsBytesRead", stats.getHdfsBytesRead());
    metrics.put("hdfsBytesWritten", stats.getHdfsBytesWritten());
    metrics.put("bytesWritten", stats.getBytesWritten());
    metrics.put("recordWritten", stats.getRecordWrittern());
    if (stats instanceof MRJobStats) {
        MRJobStats mrStats = (MRJobStats) stats;
        setCounterGroupMap(CounterGroup.counterGroupsByName(mrStats.getHadoopCounters()));
        metrics.put("avgMapTime", mrStats.getAvgMapTime());
        // internal pig seems to have fixed typo in OSS pig method name; avoid NoSuchMethodException
        // TODO: Remove this once internal pig is replaced w/ OSS pig
        Number avgReduceTime = null;
        try {
            Method method = mrStats.getClass().getMethod("getAvgReduceTime");
            avgReduceTime = (Number) method.invoke(mrStats);
        } catch (NoSuchMethodException e) {
        // assume we're dealing with OSS pig; ignore
        } catch (InvocationTargetException e) {
            throw new RuntimeException("Failed to invoke MRJobStats.getAvgReduceTime", e);
        } catch (IllegalAccessException e) {
            throw new RuntimeException("Failed to invoke MRJobStats.getAvgReduceTime", e);
        }
        if (avgReduceTime == null) {
            avgReduceTime = mrStats.getAvgREduceTime();
        }
        metrics.put("avgReduceTime", avgReduceTime);
        metrics.put("mapInputRecords", mrStats.getMapInputRecords());
        metrics.put("mapOutputRecords", mrStats.getMapOutputRecords());
        metrics.put("maxMapTime", mrStats.getMaxMapTime());
        metrics.put("maxReduceTime", mrStats.getMaxReduceTime());
        metrics.put("minMapTime", mrStats.getMinMapTime());
        metrics.put("minReduceTime", mrStats.getMinReduceTime());
        metrics.put("numberMaps", mrStats.getNumberMaps());
        metrics.put("numberReduces", mrStats.getNumberReduces());
        metrics.put("proactiveSpillCountObjects", mrStats.getProactiveSpillCountObjects());
        metrics.put("proactiveSpillCountRecs", mrStats.getProactiveSpillCountRecs());
        metrics.put("reduceInputRecords", mrStats.getReduceInputRecords());
        metrics.put("reduceOutputRecords", mrStats.getReduceOutputRecords());
        metrics.put("SMMSpillCount", mrStats.getSMMSpillCount());
    }
    setMetrics(metrics);
}
Also used : MRJobStats(org.apache.pig.tools.pigstats.mapreduce.MRJobStats) Method(java.lang.reflect.Method) InvocationTargetException(java.lang.reflect.InvocationTargetException) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore)

Example 12 with JsonIgnore

use of com.fasterxml.jackson.annotation.JsonIgnore in project openstack4j by ContainX.

the class KeystoneToken method getAggregatedCatalog.

/**
     * {@inheritDoc}
     */
@Override
@JsonIgnore
public SortedSetMultimap<String, Service> getAggregatedCatalog() {
    if (aggregatedCatalog == null) {
        synchronized (this) {
            if (aggregatedCatalog == null) {
                aggregatedCatalog = TreeMultimap.create();
                for (Service sc : catalog) {
                    String nameKey = TYPE_WITHOUT_VERSION.apply(sc.getName());
                    String typeKey = TYPE_WITHOUT_VERSION.apply(sc.getType());
                    aggregatedCatalog.put(nameKey, sc);
                    aggregatedCatalog.put(typeKey, sc);
                }
            }
        }
    }
    return aggregatedCatalog;
}
Also used : Service(org.openstack4j.model.identity.v3.Service) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore)

Example 13 with JsonIgnore

use of com.fasterxml.jackson.annotation.JsonIgnore in project shifu by ShifuML.

the class EvalConfig method getScoreMetaColumns.

@JsonIgnore
public List<String> getScoreMetaColumns(ModelConfig modelConfig) throws IOException {
    if (scoreMetaColumns == null) {
        synchronized (this) {
            if (scoreMetaColumns == null) {
                if (StringUtils.isNotBlank(scoreMetaColumnNameFile)) {
                    String path = scoreMetaColumnNameFile;
                    if (SourceType.HDFS.equals(dataSet.getSource())) {
                        PathFinder pathFinder = new PathFinder(modelConfig);
                        File file = new File(scoreMetaColumnNameFile);
                        path = new Path(pathFinder.getEvalSetPath(this), file.getName()).toString();
                    }
                    String delimiter = StringUtils.isBlank(dataSet.getHeaderDelimiter()) ? dataSet.getDataDelimiter() : dataSet.getHeaderDelimiter();
                    scoreMetaColumns = CommonUtils.readConfFileIntoList(path, dataSet.getSource(), delimiter);
                }
                if (this.scoreMetaColumns == null) {
                    this.scoreMetaColumns = new ArrayList<String>();
                }
            }
        }
    }
    return scoreMetaColumns;
}
Also used : Path(org.apache.hadoop.fs.Path) PathFinder(ml.shifu.shifu.fs.PathFinder) File(java.io.File) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore)

Example 14 with JsonIgnore

use of com.fasterxml.jackson.annotation.JsonIgnore in project shifu by ShifuML.

the class EvalConfig method getAllMetaColumns.

@JsonIgnore
public List<String> getAllMetaColumns(ModelConfig modelConfig) throws IOException {
    if (metaColumns == null) {
        synchronized (this) {
            if (metaColumns == null) {
                List<String> scoreMetaColumns = getScoreMetaColumns(modelConfig);
                if (scoreMetaColumns != null) {
                    this.metaColumns = new ArrayList<String>(scoreMetaColumns);
                }
                String metaColumnNameFile = dataSet.getMetaColumnNameFile();
                if (StringUtils.isNotBlank(metaColumnNameFile)) {
                    String path = metaColumnNameFile;
                    if (SourceType.HDFS.equals(dataSet.getSource())) {
                        PathFinder pathFinder = new PathFinder(modelConfig);
                        File file = new File(metaColumnNameFile);
                        path = new Path(pathFinder.getEvalSetPath(this), file.getName()).toString();
                    }
                    String delimiter = StringUtils.isBlank(dataSet.getHeaderDelimiter()) ? dataSet.getDataDelimiter() : dataSet.getHeaderDelimiter();
                    List<String> rawMetaColumns = CommonUtils.readConfFileIntoList(path, dataSet.getSource(), delimiter);
                    if (CollectionUtils.isNotEmpty(metaColumns)) {
                        for (String column : rawMetaColumns) {
                            if (!metaColumns.contains(column)) {
                                metaColumns.add(column);
                            }
                        }
                    } else {
                        metaColumns = rawMetaColumns;
                    }
                }
                if (this.metaColumns == null) {
                    this.metaColumns = new ArrayList<String>();
                }
            }
        }
    }
    return metaColumns;
}
Also used : Path(org.apache.hadoop.fs.Path) PathFinder(ml.shifu.shifu.fs.PathFinder) File(java.io.File) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore)

Example 15 with JsonIgnore

use of com.fasterxml.jackson.annotation.JsonIgnore in project XRTB by benmfaul.

the class Node method getLucene.

@JsonIgnore
public String getLucene() {
    String stuff = "";
    if (value.toString().startsWith("@") == true)
        return null;
    String hr = this.hierarchy.replace("imp.0.", "imp.");
    hr = hr.replaceAll("exchange", "ext.exchange");
    if (this.notPresentOk == true) {
        stuff = "((-_exists_: " + hr + ") OR ";
    }
    String strValue = value.toString();
    strValue = strValue.replaceAll("/", "\\\\/");
    switch(operator) {
        case QUERY:
            return null;
        case EQUALS:
            stuff += hr + ": " + strValue;
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case NOT_EQUALS:
            stuff += "-" + hr + ": " + strValue;
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case STRINGIN:
            stuff += hr + ": \"" + strValue + "\"";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case NOT_STRINGIN:
            stuff += "-" + hr + ": \"" + strValue + "\"";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case NOT_INTERSECTS:
            if (value instanceof List) {
                String str = "(";
                List list = (List) value;
                for (int i = 0; i < list.size(); i++) {
                    str += "-" + hr + ": *" + list.get(i) + "*";
                    str = str.replaceAll("/", "\\\\/");
                    if (i + 1 < list.size()) {
                        str += " OR ";
                    }
                }
                str += ")";
                stuff += str;
                if (notPresentOk)
                    stuff += ")";
                return stuff;
            }
            stuff += "-" + hr + ": " + strValue;
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case INTERSECTS:
            if (value instanceof List) {
                String str = "(";
                List list = (List) value;
                for (int i = 0; i < list.size(); i++) {
                    str += hr + ": *" + list.get(i) + "*";
                    str = str.replaceAll("/", "\\\\/");
                    if (i + 1 < list.size()) {
                        str += " OR ";
                    }
                }
                str += ")";
                stuff += str;
                if (notPresentOk)
                    stuff += ")";
                return stuff;
            }
            stuff += hr + ": *" + strValue + "*";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case MEMBER:
            if (value instanceof List) {
                String str = "(";
                List list = (List) value;
                for (int i = 0; i < list.size(); i++) {
                    str += hr + ": *" + list.get(i) + "*";
                    str = str.replaceAll("/", "\\\\/");
                    if (i + 1 < list.size()) {
                        str += " OR ";
                    }
                }
                str += ")";
                stuff += str;
                if (notPresentOk)
                    stuff += ")";
                return stuff;
            }
            stuff += hr + ": *" + strValue + "*";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case NOT_MEMBER:
            if (value instanceof List) {
                String str = "(";
                List list = (List) value;
                for (int i = 0; i < list.size(); i++) {
                    str += "-" + hr + ": *" + list.get(i) + "*";
                    str = str.replaceAll("/", "\\\\/");
                    if (i + 1 < list.size()) {
                        str += " OR ";
                    }
                }
                str += ")";
                stuff += str;
                if (notPresentOk)
                    stuff += ")";
                return stuff;
            }
            stuff += "-" + hr + ": *" + strValue + "*";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case INRANGE:
            List o = (List) value;
            stuff += hr + ": [" + o.get(0) + " TO " + o.get(1) + "]";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case NOT_INRANGE:
            List list = (List) value;
            stuff += "-" + hr + ": [" + list.get(0) + " TO " + list.get(1) + "]";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case DOMAIN:
            list = (List) value;
            stuff += "(" + hr + "< " + list.get(1) + " AND " + hr + "> " + list.get(0) + ")";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case NOT_DOMAIN:
            list = (List) value;
            stuff += "(" + hr + "> " + list.get(1) + " OR " + hr + "< " + list.get(0) + ")";
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case LESS_THAN:
            stuff += hr + "< " + strValue;
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case LESS_THAN_EQUALS:
            stuff += hr + "<= " + strValue;
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case GREATER_THAN:
            stuff = hr + "< " + strValue;
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case GREATER_THAN_EQUALS:
            stuff += hr + ">= " + strValue;
            if (notPresentOk)
                stuff += ")";
            return stuff;
        case EXISTS:
            return "_exists_: " + hr;
        case NOT_EXISTS:
            return "_missing_: " + hr;
    }
    return "";
}
Also used : ArrayList(java.util.ArrayList) List(java.util.List) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore)

Aggregations

JsonIgnore (com.fasterxml.jackson.annotation.JsonIgnore)15 EntitlementException (com.sun.identity.entitlement.EntitlementException)3 PolicyException (com.sun.identity.policy.PolicyException)3 SSOException (com.iplanet.sso.SSOException)2 File (java.io.File)2 PathFinder (ml.shifu.shifu.fs.PathFinder)2 Path (org.apache.hadoop.fs.Path)2 Counters (org.apache.hadoop.mapred.Counters)2 JSONException (org.json.JSONException)2 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)1 EntitlementSubject (com.sun.identity.entitlement.EntitlementSubject)1 Condition (com.sun.identity.policy.interfaces.Condition)1 ResponseProvider (com.sun.identity.policy.interfaces.ResponseProvider)1 Subject (com.sun.identity.policy.interfaces.Subject)1 ResourceType (com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage.ResourceType)1 DimensionsSpec (io.druid.data.input.impl.DimensionsSpec)1 InputRowParser (io.druid.data.input.impl.InputRowParser)1 TimestampSpec (io.druid.data.input.impl.TimestampSpec)1 IAE (io.druid.java.util.common.IAE)1 AggregatorFactory (io.druid.query.aggregation.AggregatorFactory)1