Search in sources :

Example 1 with Combiner

use of uk.me.parabola.mkgmap.combiners.Combiner in project mkgmap by openstreetmap.

the class Main method endOptions.

public void endOptions(CommandArgs args) {
    fileOptions(args);
    log.info("Start tile processors");
    int threadCount = maxJobs;
    int taskCount = futures.size();
    Runtime runtime = Runtime.getRuntime();
    if (threadPool == null) {
        if (threadCount == 0) {
            threadCount = 1;
            if (taskCount > 2) {
                // run one task to see how much memory it uses
                log.info("Max Memory: " + runtime.maxMemory());
                futures.get(0).run();
                long maxMemory = 0;
                for (MemoryPoolMXBean mxBean : ManagementFactory.getMemoryPoolMXBeans()) {
                    if (mxBean.getType() == MemoryType.HEAP) {
                        MemoryUsage memoryUsage = mxBean.getPeakUsage();
                        log.info("Max: " + memoryUsage.getMax());
                        log.info("Used: " + memoryUsage.getUsed());
                        if (memoryUsage.getMax() > maxMemory && memoryUsage.getUsed() != 0) {
                            maxMemory = memoryUsage.getMax();
                            threadCount = (int) (memoryUsage.getMax() / memoryUsage.getUsed());
                        }
                    }
                }
                threadCount = Math.max(threadCount, 1);
                threadCount = Math.min(threadCount, runtime.availableProcessors());
                System.out.println("Setting max-jobs to " + threadCount);
            }
        }
        log.info("Creating thread pool with " + threadCount + " threads");
        threadPool = Executors.newFixedThreadPool(threadCount);
    }
    // process all input files
    for (FilenameTask task : futures) {
        threadPool.execute(task);
    }
    List<FilenameTask> filenames = new ArrayList<>();
    int numMapFailedExceptions = 0;
    if (threadPool != null) {
        threadPool.shutdown();
        while (!futures.isEmpty()) {
            try {
                try {
                    // don't call get() until a job has finished
                    if (futures.get(0).isDone()) {
                        FilenameTask future = futures.remove(0);
                        // Provoke any exceptions by calling get and then
                        // save the result for later use
                        future.setFilename(future.get());
                        filenames.add(future);
                    } else
                        Thread.sleep(100);
                } catch (ExecutionException e) {
                    // Re throw the underlying exception
                    Throwable cause = e.getCause();
                    if (cause instanceof Exception)
                        // noinspection ProhibitedExceptionThrown
                        throw (Exception) cause;
                    else if (cause instanceof Error)
                        // noinspection ProhibitedExceptionThrown
                        throw (Error) cause;
                    else
                        throw e;
                }
            } catch (OutOfMemoryError | ExitException e) {
                throw e;
            } catch (MapFailedException mfe) {
                // System.err.println(mfe.getMessage()); // already printed via log
                numMapFailedExceptions++;
                setProgramRC(-1);
            } catch (Throwable t) {
                t.printStackTrace();
                if (!args.getProperties().getProperty("keep-going", false)) {
                    throw new ExitException("Exiting - if you want to carry on regardless, use the --keep-going option");
                }
            }
        }
    }
    System.out.println("Number of MapFailedExceptions: " + numMapFailedExceptions);
    if ((taskCount > threadCount + 1) && (maxJobs == 0) && (threadCount < runtime.availableProcessors())) {
        System.out.println("To reduce the run time, consider increasing the amnount of memory available for use by mkgmap by using the Java -Xmx flag to set the memory to more than " + 100 * (1 + ((runtime.maxMemory() * runtime.availableProcessors()) / (threadCount * 1024 * 1024 * 100))) + " MB, providing this is less than the amount of physical memory installed.");
    }
    if (combiners.isEmpty())
        return;
    boolean hasFiles = false;
    for (FilenameTask file : filenames) {
        if (file == null || file.isCancelled() || file.getFilename() == null) {
            if (args.getProperties().getProperty("keep-going", false))
                continue;
            else
                throw new ExitException("Exiting - if you want to carry on regardless, use the --keep-going option");
        }
        hasFiles = true;
    }
    if (!hasFiles) {
        log.warn("nothing to do for combiners.");
        return;
    }
    log.info("Combining maps");
    args.setSort(getSort(args));
    // Get them all set up.
    for (Combiner c : combiners) c.init(args);
    filenames.sort(new Comparator<FilenameTask>() {

        public int compare(FilenameTask o1, FilenameTask o2) {
            if (!o1.getFilename().endsWith(".img") || !o2.getFilename().endsWith(".img"))
                return o1.getFilename().compareTo(o2.getFilename());
            // Both end in .img
            try {
                int id1 = FileInfo.getFileInfo(o1.getFilename()).getHexname();
                int id2 = FileInfo.getFileInfo(o2.getFilename()).getHexname();
                if (id1 == id2)
                    return 0;
                else if (id1 < id2)
                    return -1;
                else
                    return 1;
            } catch (FileNotFoundException ignored) {
            }
            return 0;
        }
    });
    // will contain img files for which an additional ovm file was found
    HashSet<String> foundOvmFiles = new HashSet<>();
    // try OverviewBuilder with special files
    if (tdbBuilderAdded) {
        for (FilenameTask file : filenames) {
            if (file == null || file.isCancelled())
                continue;
            try {
                String fileName = file.getFilename();
                if (!fileName.endsWith(".img"))
                    continue;
                fileName = OverviewBuilder.getOverviewImgName(fileName);
                log.info("  " + fileName);
                FileInfo fileInfo = FileInfo.getFileInfo(fileName);
                fileInfo.setArgs(file.getArgs());
                // add the real input file
                foundOvmFiles.add(file.getFilename());
                for (Combiner c : combiners) {
                    if (c instanceof OverviewBuilder)
                        c.onMapEnd(fileInfo);
                }
            } catch (FileNotFoundException ignored) {
            }
        }
    }
    // Tell them about each filename (OverviewBuilder excluded)
    for (FilenameTask file : filenames) {
        if (file == null || file.isCancelled())
            continue;
        try {
            log.info("  " + file);
            FileInfo fileInfo = FileInfo.getFileInfo(file.getFilename());
            fileInfo.setArgs(file.getArgs());
            for (Combiner c : combiners) {
                if (c instanceof OverviewBuilder && foundOvmFiles.contains(file.getFilename()))
                    continue;
                c.onMapEnd(fileInfo);
            }
        } catch (FileNotFoundException e) {
            throw new MapFailedException("could not open file " + e.getMessage());
        }
    }
    // All done, allow tidy up or file creation to happen
    for (Combiner c : combiners) c.onFinish();
    if (tdbBuilderAdded && args.getProperties().getProperty("remove-ovm-work-files", false)) {
        for (String fName : foundOvmFiles) {
            String ovmFile = OverviewBuilder.getOverviewImgName(fName);
            log.info("removing " + ovmFile);
            new File(ovmFile).delete();
        }
    }
}
Also used : MapFailedException(uk.me.parabola.imgfmt.MapFailedException) ArrayList(java.util.ArrayList) FileNotFoundException(java.io.FileNotFoundException) FileInfo(uk.me.parabola.mkgmap.combiners.FileInfo) ExecutionException(java.util.concurrent.ExecutionException) ExitException(uk.me.parabola.imgfmt.ExitException) HashSet(java.util.HashSet) OutOfMemoryError(java.lang.OutOfMemoryError) Combiner(uk.me.parabola.mkgmap.combiners.Combiner) MemoryUsage(java.lang.management.MemoryUsage) OverviewBuilder(uk.me.parabola.mkgmap.combiners.OverviewBuilder) FileNotFoundException(java.io.FileNotFoundException) ExitException(uk.me.parabola.imgfmt.ExitException) SyntaxException(uk.me.parabola.mkgmap.scan.SyntaxException) MapFailedException(uk.me.parabola.imgfmt.MapFailedException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) OutOfMemoryError(java.lang.OutOfMemoryError) MemoryPoolMXBean(java.lang.management.MemoryPoolMXBean) File(java.io.File)

Aggregations

File (java.io.File)1 FileNotFoundException (java.io.FileNotFoundException)1 IOException (java.io.IOException)1 OutOfMemoryError (java.lang.OutOfMemoryError)1 MemoryPoolMXBean (java.lang.management.MemoryPoolMXBean)1 MemoryUsage (java.lang.management.MemoryUsage)1 ArrayList (java.util.ArrayList)1 HashSet (java.util.HashSet)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExitException (uk.me.parabola.imgfmt.ExitException)1 MapFailedException (uk.me.parabola.imgfmt.MapFailedException)1 Combiner (uk.me.parabola.mkgmap.combiners.Combiner)1 FileInfo (uk.me.parabola.mkgmap.combiners.FileInfo)1 OverviewBuilder (uk.me.parabola.mkgmap.combiners.OverviewBuilder)1 SyntaxException (uk.me.parabola.mkgmap.scan.SyntaxException)1