Examples with MapFailedException uk.me.parabola.imgfmt.MapFailedException used on opensource projects

Example 1 with MapFailedException

use of uk.me.parabola.imgfmt.MapFailedException in project mkgmap by openstreetmap.

the class Main method endOptions.

public void endOptions(CommandArgs args) {
    fileOptions(args);
    log.info("Start tile processors");
    int threadCount = maxJobs;
    int taskCount = futures.size();
    Runtime runtime = Runtime.getRuntime();
    if (threadPool == null) {
        if (threadCount == 0) {
            threadCount = 1;
            if (taskCount > 2) {
                // run one task to see how much memory it uses
                log.info("Max Memory: " + runtime.maxMemory());
                futures.get(0).run();
                long maxMemory = 0;
                for (MemoryPoolMXBean mxBean : ManagementFactory.getMemoryPoolMXBeans()) {
                    if (mxBean.getType() == MemoryType.HEAP) {
                        MemoryUsage memoryUsage = mxBean.getPeakUsage();
                        log.info("Max: " + memoryUsage.getMax());
                        log.info("Used: " + memoryUsage.getUsed());
                        if (memoryUsage.getMax() > maxMemory && memoryUsage.getUsed() != 0) {
                            maxMemory = memoryUsage.getMax();
                            threadCount = (int) (memoryUsage.getMax() / memoryUsage.getUsed());
                        }
                    }
                }
                threadCount = Math.max(threadCount, 1);
                threadCount = Math.min(threadCount, runtime.availableProcessors());
                System.out.println("Setting max-jobs to " + threadCount);
            }
        }
        log.info("Creating thread pool with " + threadCount + " threads");
        threadPool = Executors.newFixedThreadPool(threadCount);
    }
    // process all input files
    for (FilenameTask task : futures) {
        threadPool.execute(task);
    }
    List<FilenameTask> filenames = new ArrayList<>();
    int numMapFailedExceptions = 0;
    if (threadPool != null) {
        threadPool.shutdown();
        while (!futures.isEmpty()) {
            try {
                try {
                    // don't call get() until a job has finished
                    if (futures.get(0).isDone()) {
                        FilenameTask future = futures.remove(0);
                        // Provoke any exceptions by calling get and then
                        // save the result for later use
                        future.setFilename(future.get());
                        filenames.add(future);
                    } else
                        Thread.sleep(100);
                } catch (ExecutionException e) {
                    // Re throw the underlying exception
                    Throwable cause = e.getCause();
                    if (cause instanceof Exception)
                        // noinspection ProhibitedExceptionThrown
                        throw (Exception) cause;
                    else if (cause instanceof Error)
                        // noinspection ProhibitedExceptionThrown
                        throw (Error) cause;
                    else
                        throw e;
                }
            } catch (OutOfMemoryError | ExitException e) {
                throw e;
            } catch (MapFailedException mfe) {
                // System.err.println(mfe.getMessage()); // already printed via log
                numMapFailedExceptions++;
                setProgramRC(-1);
            } catch (Throwable t) {
                t.printStackTrace();
                if (!args.getProperties().getProperty("keep-going", false)) {
                    throw new ExitException("Exiting - if you want to carry on regardless, use the --keep-going option");
                }
            }
        }
    }
    System.out.println("Number of MapFailedExceptions: " + numMapFailedExceptions);
    if ((taskCount > threadCount + 1) && (maxJobs == 0) && (threadCount < runtime.availableProcessors())) {
        System.out.println("To reduce the run time, consider increasing the amnount of memory available for use by mkgmap by using the Java -Xmx flag to set the memory to more than " + 100 * (1 + ((runtime.maxMemory() * runtime.availableProcessors()) / (threadCount * 1024 * 1024 * 100))) + " MB, providing this is less than the amount of physical memory installed.");
    }
    if (combiners.isEmpty())
        return;
    boolean hasFiles = false;
    for (FilenameTask file : filenames) {
        if (file == null || file.isCancelled() || file.getFilename() == null) {
            if (args.getProperties().getProperty("keep-going", false))
                continue;
            else
                throw new ExitException("Exiting - if you want to carry on regardless, use the --keep-going option");
        }
        hasFiles = true;
    }
    if (!hasFiles) {
        log.warn("nothing to do for combiners.");
        return;
    }
    log.info("Combining maps");
    args.setSort(getSort(args));
    // Get them all set up.
    for (Combiner c : combiners) c.init(args);
    filenames.sort(new Comparator<FilenameTask>() {

        public int compare(FilenameTask o1, FilenameTask o2) {
            if (!o1.getFilename().endsWith(".img") || !o2.getFilename().endsWith(".img"))
                return o1.getFilename().compareTo(o2.getFilename());
            // Both end in .img
            try {
                int id1 = FileInfo.getFileInfo(o1.getFilename()).getHexname();
                int id2 = FileInfo.getFileInfo(o2.getFilename()).getHexname();
                if (id1 == id2)
                    return 0;
                else if (id1 < id2)
                    return -1;
                else
                    return 1;
            } catch (FileNotFoundException ignored) {
            }
            return 0;
        }
    });
    // will contain img files for which an additional ovm file was found
    HashSet<String> foundOvmFiles = new HashSet<>();
    // try OverviewBuilder with special files
    if (tdbBuilderAdded) {
        for (FilenameTask file : filenames) {
            if (file == null || file.isCancelled())
                continue;
            try {
                String fileName = file.getFilename();
                if (!fileName.endsWith(".img"))
                    continue;
                fileName = OverviewBuilder.getOverviewImgName(fileName);
                log.info("  " + fileName);
                FileInfo fileInfo = FileInfo.getFileInfo(fileName);
                fileInfo.setArgs(file.getArgs());
                // add the real input file
                foundOvmFiles.add(file.getFilename());
                for (Combiner c : combiners) {
                    if (c instanceof OverviewBuilder)
                        c.onMapEnd(fileInfo);
                }
            } catch (FileNotFoundException ignored) {
            }
        }
    }
    // Tell them about each filename (OverviewBuilder excluded)
    for (FilenameTask file : filenames) {
        if (file == null || file.isCancelled())
            continue;
        try {
            log.info("  " + file);
            FileInfo fileInfo = FileInfo.getFileInfo(file.getFilename());
            fileInfo.setArgs(file.getArgs());
            for (Combiner c : combiners) {
                if (c instanceof OverviewBuilder && foundOvmFiles.contains(file.getFilename()))
                    continue;
                c.onMapEnd(fileInfo);
            }
        } catch (FileNotFoundException e) {
            throw new MapFailedException("could not open file " + e.getMessage());
        }
    }
    // All done, allow tidy up or file creation to happen
    for (Combiner c : combiners) c.onFinish();
    if (tdbBuilderAdded && args.getProperties().getProperty("remove-ovm-work-files", false)) {
        for (String fName : foundOvmFiles) {
            String ovmFile = OverviewBuilder.getOverviewImgName(fName);
            log.info("removing " + ovmFile);
            new File(ovmFile).delete();
        }
    }
}

Example 2 with MapFailedException

use of uk.me.parabola.imgfmt.MapFailedException in project mkgmap by openstreetmap.

the class TypCompiler method makeMap.

/**
 * The integration with mkgmap.
 *
 * @param args The options that are in force.
 * @param filename The input filename.
 * @return Returns the name of the file that was written. It depends on the family id.
 */
public String makeMap(CommandArgs args, String filename) {
    assert filename.toLowerCase().endsWith(".txt");
    CharsetProbe probe = new CharsetProbe();
    String readCharset = probe.probeCharset(filename);
    TypData data;
    try {
        data = compile(filename, readCharset, args.getSort());
    } catch (SyntaxException e) {
        throw new MapFailedException("Compiling TYP txt file: " + e.getMessage());
    } catch (FileNotFoundException e) {
        throw new MapFailedException("Could not open TYP file " + filename + " to read");
    }
    TypParam param = data.getParam();
    int family = args.get("family-id", -1);
    int product = args.get("product-id", -1);
    int cp = args.get("code-page", -1);
    if (family != -1)
        param.setFamilyId(family);
    if (product != -1)
        param.setProductId(product);
    if (cp != -1)
        param.setCodePage(cp);
    File outFile = new File(filename);
    String outName = outFile.getName();
    int last;
    if (outName.length() > 4 && (last = outName.lastIndexOf('.')) > 0)
        outName = outName.substring(0, last);
    outName += ".typ";
    outFile = new File(args.getOutputDir(), outName);
    try {
        writeTyp(data, outFile);
    } catch (TypLabelException e) {
        throw new MapFailedException("TYP file cannot be written in code page " + data.getSort().getCodepage());
    } catch (IOException e) {
        throw new MapFailedException("Error while writing typ file", e);
    }
    return outFile.getPath();
}

Example 3 with MapFailedException

use of uk.me.parabola.imgfmt.MapFailedException in project mkgmap by openstreetmap.

the class DEMTile method writeValAsBin.

/**
 * Write an unsigned binary value with the given number of bits, MSB first.
 * @param val
 * @param numBits
 */
private void writeValAsBin(int val, int numBits) {
    if (numBits == 0 && val == 0)
        return;
    int t = 1 << (numBits - 1);
    if (val >= t << 1)
        throw new MapFailedException("Number too big for binary encoding with " + numBits + " bits:" + val);
    while (t > 0) {
        addBit((val & t) != 0);
        t >>= 1;
    }
}

Example 4 with MapFailedException

use of uk.me.parabola.imgfmt.MapFailedException in project mkgmap by openstreetmap.

the class FileBackedImgFileWriter method position.

/**
 * Set the position of the file.
 * The buffer has to be flushed first.
 *
 * @param pos The new position in the file.
 */
public void position(long pos) {
    try {
        file.flush();
        tmpChannel.position(pos);
    } catch (IOException e) {
        throw new MapFailedException("Could not set position in mdr tmp file");
    }
}

Example 5 with MapFailedException

use of uk.me.parabola.imgfmt.MapFailedException in project mkgmap by openstreetmap.

the class FileBackedImgFileWriter method put3.

/**
 * Write out three bytes.  Done in the little endian byte order.
 *
 * @param val The value to write, only the bottom three bytes will be written.
 */
public void put3(int val) {
    try {
        file.write(val);
        file.write(val >> 8);
        file.write(val >> 16);
    } catch (IOException e) {
        throw new MapFailedException("could not write3 to mdr tmp file");
    }
}