Examples with TreeMap java.util.TreeMap used on opensource projects

Example 66 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class SliveTest method writeReport.

/**
   * Attempts to write the report to the given output using the specified
   * config. It will open up the expected reducer output file and read in its
   * contents and then split up by operation output and sort by operation type
   * and then for each operation type it will generate a report to the specified
   * result file and the console.
   * 
   * @param cfg
   *          the config specifying the files and output
   * 
   * @throws Exception
   *           if files can not be opened/closed/read or invalid format
   */
private void writeReport(ConfigExtractor cfg) throws Exception {
    Path dn = cfg.getOutputPath();
    LOG.info("Writing report using contents of " + dn);
    FileSystem fs = dn.getFileSystem(cfg.getConfig());
    FileStatus[] reduceFiles = fs.listStatus(dn);
    BufferedReader fileReader = null;
    PrintWriter reportWriter = null;
    try {
        List<OperationOutput> noOperations = new ArrayList<OperationOutput>();
        Map<String, List<OperationOutput>> splitTypes = new TreeMap<String, List<OperationOutput>>();
        for (FileStatus fn : reduceFiles) {
            if (!fn.getPath().getName().startsWith("part"))
                continue;
            fileReader = new BufferedReader(new InputStreamReader(new DataInputStream(fs.open(fn.getPath()))));
            String line;
            while ((line = fileReader.readLine()) != null) {
                String[] pieces = line.split("\t", 2);
                if (pieces.length == 2) {
                    OperationOutput data = new OperationOutput(pieces[0], pieces[1]);
                    String op = (data.getOperationType());
                    if (op != null) {
                        List<OperationOutput> opList = splitTypes.get(op);
                        if (opList == null) {
                            opList = new ArrayList<OperationOutput>();
                        }
                        opList.add(data);
                        splitTypes.put(op, opList);
                    } else {
                        noOperations.add(data);
                    }
                } else {
                    throw new IOException("Unparseable line " + line);
                }
            }
            fileReader.close();
            fileReader = null;
        }
        File resFile = null;
        if (cfg.getResultFile() != null) {
            resFile = new File(cfg.getResultFile());
        }
        if (resFile != null) {
            LOG.info("Report results being placed to logging output and to file " + resFile.getCanonicalPath());
            reportWriter = new PrintWriter(new FileOutputStream(resFile));
        } else {
            LOG.info("Report results being placed to logging output");
        }
        ReportWriter reporter = new ReportWriter();
        if (!noOperations.isEmpty()) {
            reporter.basicReport(noOperations, reportWriter);
        }
        for (String opType : splitTypes.keySet()) {
            reporter.opReport(opType, splitTypes.get(opType), reportWriter);
        }
    } finally {
        if (fileReader != null) {
            fileReader.close();
        }
        if (reportWriter != null) {
            reportWriter.close();
        }
    }
}

Example 67 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class HadoopArchives method writeTopLevelDirs.

/**
   * this method writes all the valid top level directories 
   * into the srcWriter for indexing. This method is a little
   * tricky. example- 
   * for an input with parent path /home/user/ and sources 
   * as /home/user/source/dir1, /home/user/source/dir2 - this 
   * will output <source, dir, dir1, dir2> (dir means that source is a dir
   * with dir1 and dir2 as children) and <source/dir1, file, null>
   * and <source/dir2, file, null>
   * @param srcWriter the sequence file writer to write the
   * directories to
   * @param paths the source paths provided by the user. They
   * are glob free and have full path (not relative paths)
   * @param parentPath the parent path that you want the archives
   * to be relative to. example - /home/user/dir1 can be archived with
   * parent as /home or /home/user.
   * @throws IOException
   */
private void writeTopLevelDirs(SequenceFile.Writer srcWriter, List<Path> paths, Path parentPath) throws IOException {
    // extract paths from absolute URI's
    List<Path> justPaths = new ArrayList<Path>();
    for (Path p : paths) {
        justPaths.add(new Path(p.toUri().getPath()));
    }
    /* find all the common parents of paths that are valid archive
     * paths. The below is done so that we do not add a common path
     * twice and also we need to only add valid child of a path that
     * are specified the user.
     */
    TreeMap<String, HashSet<String>> allpaths = new TreeMap<String, HashSet<String>>();
    /* the largest depth of paths. the max number of times
     * we need to iterate
     */
    Path deepest = largestDepth(paths);
    Path root = new Path(Path.SEPARATOR);
    for (int i = parentPath.depth(); i < deepest.depth(); i++) {
        List<Path> parents = new ArrayList<Path>();
        for (Path p : justPaths) {
            if (p.compareTo(root) == 0) {
            //do nothing
            } else {
                Path parent = p.getParent();
                if (null != parent) {
                    if (allpaths.containsKey(parent.toString())) {
                        HashSet<String> children = allpaths.get(parent.toString());
                        children.add(p.getName());
                    } else {
                        HashSet<String> children = new HashSet<String>();
                        children.add(p.getName());
                        allpaths.put(parent.toString(), children);
                    }
                    parents.add(parent);
                }
            }
        }
        justPaths = parents;
    }
    Set<Map.Entry<String, HashSet<String>>> keyVals = allpaths.entrySet();
    for (Map.Entry<String, HashSet<String>> entry : keyVals) {
        final Path relPath = relPathToRoot(new Path(entry.getKey()), parentPath);
        if (relPath != null) {
            final String[] children = new String[entry.getValue().size()];
            int i = 0;
            for (String child : entry.getValue()) {
                children[i++] = child;
            }
            append(srcWriter, 0L, relPath.toString(), children);
        }
    }
}

Example 68 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class RollingLevelDBTimelineStore method getEntityTimelines.

@Override
public TimelineEvents getEntityTimelines(String entityType, SortedSet<String> entityIds, Long limit, Long windowStart, Long windowEnd, Set<String> eventType) throws IOException {
    TimelineEvents events = new TimelineEvents();
    if (entityIds == null || entityIds.isEmpty()) {
        return events;
    }
    // create a lexicographically-ordered map from start time to entities
    Map<byte[], List<EntityIdentifier>> startTimeMap = new TreeMap<byte[], List<EntityIdentifier>>(new Comparator<byte[]>() {

        @Override
        public int compare(byte[] o1, byte[] o2) {
            return WritableComparator.compareBytes(o1, 0, o1.length, o2, 0, o2.length);
        }
    });
    DBIterator iterator = null;
    try {
        // skip entities with no start time
        for (String entityId : entityIds) {
            byte[] startTime = getStartTime(entityId, entityType);
            if (startTime != null) {
                List<EntityIdentifier> entities = startTimeMap.get(startTime);
                if (entities == null) {
                    entities = new ArrayList<EntityIdentifier>();
                    startTimeMap.put(startTime, entities);
                }
                entities.add(new EntityIdentifier(entityId, entityType));
            }
        }
        for (Entry<byte[], List<EntityIdentifier>> entry : startTimeMap.entrySet()) {
            // look up the events matching the given parameters (limit,
            // start time, end time, event types) for entities whose start times
            // were found and add the entities to the return list
            byte[] revStartTime = entry.getKey();
            for (EntityIdentifier entityIdentifier : entry.getValue()) {
                EventsOfOneEntity entity = new EventsOfOneEntity();
                entity.setEntityId(entityIdentifier.getId());
                entity.setEntityType(entityType);
                events.addEvent(entity);
                KeyBuilder kb = KeyBuilder.newInstance().add(entityType).add(revStartTime).add(entityIdentifier.getId()).add(EVENTS_COLUMN);
                byte[] prefix = kb.getBytesForLookup();
                if (windowEnd == null) {
                    windowEnd = Long.MAX_VALUE;
                }
                byte[] revts = writeReverseOrderedLong(windowEnd);
                kb.add(revts);
                byte[] first = kb.getBytesForLookup();
                byte[] last = null;
                if (windowStart != null) {
                    last = KeyBuilder.newInstance().add(prefix).add(writeReverseOrderedLong(windowStart)).getBytesForLookup();
                }
                if (limit == null) {
                    limit = DEFAULT_LIMIT;
                }
                DB db = entitydb.getDBForStartTime(readReverseOrderedLong(revStartTime, 0));
                if (db == null) {
                    continue;
                }
                iterator = db.iterator();
                for (iterator.seek(first); entity.getEvents().size() < limit && iterator.hasNext(); iterator.next()) {
                    byte[] key = iterator.peekNext().getKey();
                    if (!prefixMatches(prefix, prefix.length, key) || (last != null && WritableComparator.compareBytes(key, 0, key.length, last, 0, last.length) > 0)) {
                        break;
                    }
                    TimelineEvent event = getEntityEvent(eventType, key, prefix.length, iterator.peekNext().getValue());
                    if (event != null) {
                        entity.addEvent(event);
                    }
                }
            }
        }
    } finally {
        IOUtils.cleanup(LOG, iterator);
    }
    return events;
}

Example 69 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class MiniHadoopClusterManager method start.

/**
   * Starts DFS and MR clusters, as specified in member-variable options. Also
   * writes out configuration and details, if requested.
   *
   * @throws IOException
   * @throws FileNotFoundException
   * @throws URISyntaxException
   */
public void start() throws IOException, FileNotFoundException, URISyntaxException {
    if (!noDFS) {
        dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort).nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes).startupOption(dfsOpts).build();
        LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort());
    }
    if (!noMR) {
        if (fs == null && dfs != null) {
            fs = dfs.getFileSystem().getUri().toString();
        } else if (fs == null) {
            fs = "file:///tmp/minimr-" + System.nanoTime();
        }
        FileSystem.setDefaultUri(conf, new URI(fs));
        // Instruct the minicluster to use fixed ports, so user will know which
        // ports to use when communicating with the cluster.
        conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
        conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
        conf.set(YarnConfiguration.RM_ADDRESS, MiniYARNCluster.getHostname() + ":" + this.rmPort);
        conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, MiniYARNCluster.getHostname() + ":" + this.jhsPort);
        mr = MiniMRClientClusterFactory.create(this.getClass(), numNodeManagers, conf);
        LOG.info("Started MiniMRCluster");
    }
    if (writeConfig != null) {
        FileOutputStream fos = new FileOutputStream(new File(writeConfig));
        conf.writeXml(fos);
        fos.close();
    }
    if (writeDetails != null) {
        Map<String, Object> map = new TreeMap<String, Object>();
        if (dfs != null) {
            map.put("namenode_port", dfs.getNameNodePort());
        }
        if (mr != null) {
            map.put("resourcemanager_port", mr.getConfig().get(YarnConfiguration.RM_ADDRESS).split(":")[1]);
        }
        FileWriter fw = new FileWriter(new File(writeDetails));
        fw.write(new JSON().toJSON(map));
        fw.close();
    }
}

Example 70 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class RemoteRequestsTable method remove.

ResourceRequestInfo remove(Priority priority, String resourceName, ExecutionType execType, Resource capability) {
    ResourceRequestInfo retVal = null;
    Map<String, Map<ExecutionType, TreeMap<Resource, ResourceRequestInfo>>> locationMap = remoteRequestsTable.get(priority);
    if (locationMap == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("No such priority=" + priority);
        }
        return null;
    }
    Map<ExecutionType, TreeMap<Resource, ResourceRequestInfo>> execTypeMap = locationMap.get(resourceName);
    if (execTypeMap == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("No such resourceName=" + resourceName);
        }
        return null;
    }
    TreeMap<Resource, ResourceRequestInfo> capabilityMap = execTypeMap.get(execType);
    if (capabilityMap == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("No such Execution Type=" + execType);
        }
        return null;
    }
    retVal = capabilityMap.remove(capability);
    if (capabilityMap.size() == 0) {
        execTypeMap.remove(execType);
        if (execTypeMap.size() == 0) {
            locationMap.remove(resourceName);
            if (locationMap.size() == 0) {
                this.remoteRequestsTable.remove(priority);
            }
        }
    }
    return retVal;
}