Examples with OutOfMemoryError java.lang.OutOfMemoryError used on opensource projects

Example 1 with OutOfMemoryError

use of java.lang.OutOfMemoryError in project mkgmap by openstreetmap.

the class Main method endOptions.

public void endOptions(CommandArgs args) {
    fileOptions(args);
    log.info("Start tile processors");
    int threadCount = maxJobs;
    int taskCount = futures.size();
    Runtime runtime = Runtime.getRuntime();
    if (threadPool == null) {
        if (threadCount == 0) {
            threadCount = 1;
            if (taskCount > 2) {
                // run one task to see how much memory it uses
                log.info("Max Memory: " + runtime.maxMemory());
                futures.get(0).run();
                long maxMemory = 0;
                for (MemoryPoolMXBean mxBean : ManagementFactory.getMemoryPoolMXBeans()) {
                    if (mxBean.getType() == MemoryType.HEAP) {
                        MemoryUsage memoryUsage = mxBean.getPeakUsage();
                        log.info("Max: " + memoryUsage.getMax());
                        log.info("Used: " + memoryUsage.getUsed());
                        if (memoryUsage.getMax() > maxMemory && memoryUsage.getUsed() != 0) {
                            maxMemory = memoryUsage.getMax();
                            threadCount = (int) (memoryUsage.getMax() / memoryUsage.getUsed());
                        }
                    }
                }
                threadCount = Math.max(threadCount, 1);
                threadCount = Math.min(threadCount, runtime.availableProcessors());
                System.out.println("Setting max-jobs to " + threadCount);
            }
        }
        log.info("Creating thread pool with " + threadCount + " threads");
        threadPool = Executors.newFixedThreadPool(threadCount);
    }
    // process all input files
    for (FilenameTask task : futures) {
        threadPool.execute(task);
    }
    List<FilenameTask> filenames = new ArrayList<>();
    int numMapFailedExceptions = 0;
    if (threadPool != null) {
        threadPool.shutdown();
        while (!futures.isEmpty()) {
            try {
                try {
                    // don't call get() until a job has finished
                    if (futures.get(0).isDone()) {
                        FilenameTask future = futures.remove(0);
                        // Provoke any exceptions by calling get and then
                        // save the result for later use
                        future.setFilename(future.get());
                        filenames.add(future);
                    } else
                        Thread.sleep(100);
                } catch (ExecutionException e) {
                    // Re throw the underlying exception
                    Throwable cause = e.getCause();
                    if (cause instanceof Exception)
                        // noinspection ProhibitedExceptionThrown
                        throw (Exception) cause;
                    else if (cause instanceof Error)
                        // noinspection ProhibitedExceptionThrown
                        throw (Error) cause;
                    else
                        throw e;
                }
            } catch (OutOfMemoryError | ExitException e) {
                throw e;
            } catch (MapFailedException mfe) {
                // System.err.println(mfe.getMessage()); // already printed via log
                numMapFailedExceptions++;
                setProgramRC(-1);
            } catch (Throwable t) {
                t.printStackTrace();
                if (!args.getProperties().getProperty("keep-going", false)) {
                    throw new ExitException("Exiting - if you want to carry on regardless, use the --keep-going option");
                }
            }
        }
    }
    System.out.println("Number of MapFailedExceptions: " + numMapFailedExceptions);
    if ((taskCount > threadCount + 1) && (maxJobs == 0) && (threadCount < runtime.availableProcessors())) {
        System.out.println("To reduce the run time, consider increasing the amnount of memory available for use by mkgmap by using the Java -Xmx flag to set the memory to more than " + 100 * (1 + ((runtime.maxMemory() * runtime.availableProcessors()) / (threadCount * 1024 * 1024 * 100))) + " MB, providing this is less than the amount of physical memory installed.");
    }
    if (combiners.isEmpty())
        return;
    boolean hasFiles = false;
    for (FilenameTask file : filenames) {
        if (file == null || file.isCancelled() || file.getFilename() == null) {
            if (args.getProperties().getProperty("keep-going", false))
                continue;
            else
                throw new ExitException("Exiting - if you want to carry on regardless, use the --keep-going option");
        }
        hasFiles = true;
    }
    if (!hasFiles) {
        log.warn("nothing to do for combiners.");
        return;
    }
    log.info("Combining maps");
    args.setSort(getSort(args));
    // Get them all set up.
    for (Combiner c : combiners) c.init(args);
    filenames.sort(new Comparator<FilenameTask>() {

        public int compare(FilenameTask o1, FilenameTask o2) {
            if (!o1.getFilename().endsWith(".img") || !o2.getFilename().endsWith(".img"))
                return o1.getFilename().compareTo(o2.getFilename());
            // Both end in .img
            try {
                int id1 = FileInfo.getFileInfo(o1.getFilename()).getHexname();
                int id2 = FileInfo.getFileInfo(o2.getFilename()).getHexname();
                if (id1 == id2)
                    return 0;
                else if (id1 < id2)
                    return -1;
                else
                    return 1;
            } catch (FileNotFoundException ignored) {
            }
            return 0;
        }
    });
    // will contain img files for which an additional ovm file was found
    HashSet<String> foundOvmFiles = new HashSet<>();
    // try OverviewBuilder with special files
    if (tdbBuilderAdded) {
        for (FilenameTask file : filenames) {
            if (file == null || file.isCancelled())
                continue;
            try {
                String fileName = file.getFilename();
                if (!fileName.endsWith(".img"))
                    continue;
                fileName = OverviewBuilder.getOverviewImgName(fileName);
                log.info("  " + fileName);
                FileInfo fileInfo = FileInfo.getFileInfo(fileName);
                fileInfo.setArgs(file.getArgs());
                // add the real input file
                foundOvmFiles.add(file.getFilename());
                for (Combiner c : combiners) {
                    if (c instanceof OverviewBuilder)
                        c.onMapEnd(fileInfo);
                }
            } catch (FileNotFoundException ignored) {
            }
        }
    }
    // Tell them about each filename (OverviewBuilder excluded)
    for (FilenameTask file : filenames) {
        if (file == null || file.isCancelled())
            continue;
        try {
            log.info("  " + file);
            FileInfo fileInfo = FileInfo.getFileInfo(file.getFilename());
            fileInfo.setArgs(file.getArgs());
            for (Combiner c : combiners) {
                if (c instanceof OverviewBuilder && foundOvmFiles.contains(file.getFilename()))
                    continue;
                c.onMapEnd(fileInfo);
            }
        } catch (FileNotFoundException e) {
            throw new MapFailedException("could not open file " + e.getMessage());
        }
    }
    // All done, allow tidy up or file creation to happen
    for (Combiner c : combiners) c.onFinish();
    if (tdbBuilderAdded && args.getProperties().getProperty("remove-ovm-work-files", false)) {
        for (String fName : foundOvmFiles) {
            String ovmFile = OverviewBuilder.getOverviewImgName(fName);
            log.info("removing " + ovmFile);
            new File(ovmFile).delete();
        }
    }
}

Example 2 with OutOfMemoryError

use of java.lang.OutOfMemoryError in project Anki-Android by Ramblurr.

the class Syncer method sync.

public Object[] sync(Connection con) {
    mSyncMsg = "";
    // if the deck has any pending changes, flush them first and bump mod time
    mCol.save();
    // step 1: login & metadata
    HttpResponse ret = mServer.meta();
    if (ret == null) {
        return null;
    }
    int returntype = ret.getStatusLine().getStatusCode();
    if (returntype == 403) {
        return new Object[] { "badAuth" };
    } else if (returntype != 200) {
        return new Object[] { "error", returntype, ret.getStatusLine().getReasonPhrase() };
    }
    try {
        mCol.getDb().getDatabase().beginTransaction();
        try {
            // Log.i(AnkiDroidApp.TAG, "Sync: getting meta data from server");
            JSONObject rMeta = new JSONObject(mServer.stream2String(ret.getEntity().getContent()));
            long rscm = rMeta.getLong("scm");
            int rts = rMeta.getInt("ts");
            mRMod = rMeta.getLong("mod");
            mMaxUsn = rMeta.getInt("usn");
            mMediaUsn = rMeta.getInt("musn");
            mSyncMsg = rMeta.getString("msg");
            // skip uname, AnkiDroid already stores and shows it
            if (!rMeta.getBoolean("cont")) {
                // Don't add syncMsg; it can be fetched by UI code using the accessor
                return new Object[] { "serverAbort" };
            } else {
            // don't abort, but ui should show messages after sync finishes
            // and require confirmation if it's non-empty
            }
            // Log.i(AnkiDroidApp.TAG, "Sync: building local meta data");
            JSONObject lMeta = meta();
            mLMod = lMeta.getLong("mod");
            mMinUsn = lMeta.getInt("usn");
            long lscm = lMeta.getLong("scm");
            int lts = lMeta.getInt("ts");
            long diff = Math.abs(rts - lts);
            if (diff > 300) {
                return new Object[] { "clockOff", diff };
            }
            if (mLMod == mRMod) {
                // Log.i(AnkiDroidApp.TAG, "Sync: no changes - returning");
                return new Object[] { "noChanges" };
            } else if (lscm != rscm) {
                // Log.i(AnkiDroidApp.TAG, "Sync: full sync necessary - returning");
                return new Object[] { "fullSync" };
            }
            mLNewer = mLMod > mRMod;
            // step 2: deletions
            publishProgress(con, R.string.sync_deletions_message);
            // Log.i(AnkiDroidApp.TAG, "Sync: collection removed data");
            JSONObject lrem = removed();
            JSONObject o = new JSONObject();
            o.put("minUsn", mMinUsn);
            o.put("lnewer", mLNewer);
            o.put("graves", lrem);
            // Log.i(AnkiDroidApp.TAG, "Sync: sending and receiving removed data");
            JSONObject rrem = mServer.start(o);
            if (rrem == null) {
                // Log.i(AnkiDroidApp.TAG, "Sync: error - returning");
                return null;
            }
            if (rrem.has("errorType")) {
                // Log.i(AnkiDroidApp.TAG, "Sync: error - returning");
                return new Object[] { "error", rrem.get("errorType"), rrem.get("errorReason") };
            }
            // Log.i(AnkiDroidApp.TAG, "Sync: applying removed data");
            remove(rrem);
            // ... and small objects
            publishProgress(con, R.string.sync_small_objects_message);
            // Log.i(AnkiDroidApp.TAG, "Sync: collection small changes");
            JSONObject lchg = changes();
            JSONObject sch = new JSONObject();
            sch.put("changes", lchg);
            // Log.i(AnkiDroidApp.TAG, "Sync: sending and receiving small changes");
            JSONObject rchg = mServer.applyChanges(sch);
            if (rchg == null) {
                // Log.i(AnkiDroidApp.TAG, "Sync: error - returning");
                return null;
            }
            if (rchg.has("errorType")) {
                // Log.i(AnkiDroidApp.TAG, "Sync: error - returning");
                return new Object[] { "error", rchg.get("errorType"), rchg.get("errorReason") };
            }
            // Log.i(AnkiDroidApp.TAG, "Sync: merging small changes");
            mergeChanges(lchg, rchg);
            // step 3: stream large tables from server
            publishProgress(con, R.string.sync_download_chunk);
            while (true) {
                // Log.i(AnkiDroidApp.TAG, "Sync: downloading chunked data");
                JSONObject chunk = mServer.chunk();
                if (chunk == null) {
                    // Log.i(AnkiDroidApp.TAG, "Sync: error - returning");
                    return null;
                }
                if (chunk.has("errorType")) {
                    // Log.i(AnkiDroidApp.TAG, "Sync: error - returning");
                    return new Object[] { "error", chunk.get("errorType"), chunk.get("errorReason") };
                }
                // Log.i(AnkiDroidApp.TAG, "Sync: applying chunked data");
                applyChunk(chunk);
                if (chunk.getBoolean("done")) {
                    break;
                }
            }
            // step 4: stream to server
            publishProgress(con, R.string.sync_upload_chunk);
            while (true) {
                // Log.i(AnkiDroidApp.TAG, "Sync: collecting chunked data");
                JSONObject chunk = chunk();
                JSONObject sech = new JSONObject();
                sech.put("chunk", chunk);
                // Log.i(AnkiDroidApp.TAG, "Sync: sending chunked data");
                mServer.applyChunk(sech);
                if (chunk.getBoolean("done")) {
                    break;
                }
            }
            // step 5: sanity check during beta testing
            JSONObject c = sanityCheck();
            JSONObject sanity = mServer.sanityCheck2(c);
            if (sanity == null || !sanity.optString("status", "bad").equals("ok")) {
                return new Object[] { "sanityCheckError", null };
            }
            // finalize
            publishProgress(con, R.string.sync_finish_message);
            // Log.i(AnkiDroidApp.TAG, "Sync: sending finish command");
            long mod = mServer.finish();
            if (mod == 0) {
                return new Object[] { "finishError" };
            }
            // Log.i(AnkiDroidApp.TAG, "Sync: finishing");
            finish(mod);
            publishProgress(con, R.string.sync_writing_db);
            mCol.getDb().getDatabase().setTransactionSuccessful();
        } finally {
            mCol.getDb().getDatabase().endTransaction();
        }
    } catch (JSONException e) {
        throw new RuntimeException(e);
    } catch (IllegalStateException e) {
        throw new RuntimeException(e);
    } catch (OutOfMemoryError e) {
        AnkiDroidApp.saveExceptionReportFile(e, "Syncer-sync");
        return new Object[] { "OutOfMemoryError" };
    } catch (IOException e) {
        AnkiDroidApp.saveExceptionReportFile(e, "Syncer-sync");
        return new Object[] { "IOException" };
    }
    return new Object[] { "success" };
}

Example 3 with OutOfMemoryError

use of java.lang.OutOfMemoryError in project mkgmap by openstreetmap.

the class Main method mainStart.

/**
 * The main program to make or combine maps.  We now use a two pass process,
 * first going through the arguments and make any maps and collect names
 * to be used for creating summary files like the TDB and gmapsupp.
 *
 * @param args The command line arguments.
 */
private static int mainStart(String... args) {
    Instant start = Instant.now();
    System.out.println("Time started: " + new Date());
    // We need at least one argument.
    if (args.length < 1) {
        printUsage();
        printHelp(System.err, getLang(), "options");
        return 0;
    }
    Main mm = new Main();
    int numExitExceptions = 0;
    try {
        // Read the command line arguments and process each filename found.
        CommandArgsReader commandArgs = new CommandArgsReader(mm);
        commandArgs.setValidOptions(getValidOptions(System.err));
        commandArgs.readArgs(args);
    } catch (OutOfMemoryError e) {
        ++numExitExceptions;
        System.err.println(e);
        if (mm.maxJobs > 1)
            System.err.println("Try using the mkgmap --max-jobs option with a value less than " + mm.maxJobs + " to reduce the memory requirement, or use the Java -Xmx option to increase the available heap memory.");
        else
            System.err.println("Try using the Java -Xmx option to increase the available heap memory.");
    } catch (MapFailedException e) {
        // one of the combiners failed
        e.printStackTrace();
        ++numExitExceptions;
    } catch (ExitException e) {
        ++numExitExceptions;
        String message = e.getMessage();
        Throwable cause = e.getCause();
        while (cause != null) {
            message += "\r\n" + cause.toString();
            cause = cause.getCause();
        }
        System.err.println(message);
    }
    System.out.println("Number of ExitExceptions: " + numExitExceptions);
    System.out.println("Time finished: " + new Date());
    Duration duration = Duration.between(start, Instant.now());
    long seconds = duration.getSeconds();
    if (seconds > 0) {
        long hours = seconds / 3600;
        seconds -= hours * 3600;
        long minutes = seconds / 60;
        seconds -= minutes * 60;
        System.out.println("Total time taken: " + (hours > 0 ? hours + (hours > 1 ? " hours " : " hour ") : "") + (minutes > 0 ? minutes + (minutes > 1 ? " minutes " : " minute ") : "") + (seconds > 0 ? seconds + (seconds > 1 ? " seconds" : " second") : ""));
    } else
        System.out.println("Total time taken: " + duration.getNano() / 1000000 + " ms");
    if (numExitExceptions > 0 || mm.getProgramRC() != 0) {
        return 1;
    }
    return 0;
}

Example 4 with OutOfMemoryError

use of java.lang.OutOfMemoryError in project Anki-Android by Ramblurr.

the class Connection method doInBackgroundSync.

private Payload doInBackgroundSync(Payload data) {
    // for for doInBackgroundLoadDeckCounts if any
    DeckTask.waitToFinish();
    String hkey = (String) data.data[0];
    boolean media = (Boolean) data.data[1];
    String conflictResolution = (String) data.data[2];
    int mediaUsn = (Integer) data.data[3];
    boolean colCorruptFullSync = false;
    Collection col = AnkiDroidApp.getCol();
    if (!AnkiDroidApp.colIsOpen()) {
        if (conflictResolution != null && conflictResolution.equals("download")) {
            colCorruptFullSync = true;
        } else {
            data.success = false;
            data.result = new Object[] { "genericError" };
            return data;
        }
    }
    String path = AnkiDroidApp.getCollectionPath();
    BasicHttpSyncer server = new RemoteServer(this, hkey);
    Syncer client = new Syncer(col, server);
    // run sync and check state
    boolean noChanges = false;
    if (conflictResolution == null) {
        // Log.i(AnkiDroidApp.TAG, "Sync - starting sync");
        publishProgress(R.string.sync_prepare_syncing);
        Object[] ret = client.sync(this);
        data.message = client.getSyncMsg();
        mediaUsn = client.getmMediaUsn();
        if (ret == null) {
            data.success = false;
            data.result = new Object[] { "genericError" };
            return data;
        }
        String retCode = (String) ret[0];
        if (!retCode.equals("noChanges") && !retCode.equals("success")) {
            data.success = false;
            data.result = ret;
            // note mediaUSN for later
            data.data = new Object[] { mediaUsn };
            return data;
        }
        // save and note success state
        if (retCode.equals("noChanges")) {
            // publishProgress(R.string.sync_no_changes_message);
            noChanges = true;
        } else {
        // publishProgress(R.string.sync_database_success);
        }
    } else {
        try {
            server = new FullSyncer(col, hkey, this);
            if (conflictResolution.equals("upload")) {
                // Log.i(AnkiDroidApp.TAG, "Sync - fullsync - upload collection");
                publishProgress(R.string.sync_preparing_full_sync_message);
                Object[] ret = server.upload();
                if (ret == null) {
                    data.success = false;
                    data.result = new Object[] { "genericError" };
                    AnkiDroidApp.openCollection(path);
                    return data;
                }
                if (!((String) ret[0]).equals(BasicHttpSyncer.ANKIWEB_STATUS_OK)) {
                    data.success = false;
                    data.result = ret;
                    AnkiDroidApp.openCollection(path);
                    return data;
                }
            } else if (conflictResolution.equals("download")) {
                // Log.i(AnkiDroidApp.TAG, "Sync - fullsync - download collection");
                publishProgress(R.string.sync_downloading_message);
                Object[] ret = server.download();
                if (ret == null) {
                    data.success = false;
                    data.result = new Object[] { "genericError" };
                    AnkiDroidApp.openCollection(path);
                    return data;
                }
                if (!((String) ret[0]).equals("success")) {
                    data.success = false;
                    data.result = ret;
                    if (!colCorruptFullSync) {
                        AnkiDroidApp.openCollection(path);
                    }
                    return data;
                }
            }
            col = AnkiDroidApp.openCollection(path);
        } catch (OutOfMemoryError e) {
            AnkiDroidApp.saveExceptionReportFile(e, "doInBackgroundSync-fullSync");
            data.success = false;
            data.result = new Object[] { "OutOfMemoryError" };
            data.data = new Object[] { mediaUsn };
            return data;
        } catch (RuntimeException e) {
            AnkiDroidApp.saveExceptionReportFile(e, "doInBackgroundSync-fullSync");
            data.success = false;
            data.result = new Object[] { "IOException" };
            data.data = new Object[] { mediaUsn };
            return data;
        }
    }
    // clear undo to avoid non syncing orphans (because undo resets usn too
    if (!noChanges) {
        col.clearUndo();
    }
    // then move on to media sync
    boolean noMediaChanges = false;
    String mediaError = null;
    if (media) {
        server = new RemoteMediaServer(hkey, this);
        MediaSyncer mediaClient = new MediaSyncer(col, (RemoteMediaServer) server);
        String ret;
        try {
            ret = mediaClient.sync(mediaUsn, this);
            if (ret == null) {
                mediaError = AnkiDroidApp.getAppResources().getString(R.string.sync_media_error);
            } else {
                if (ret.equals("noChanges")) {
                    publishProgress(R.string.sync_media_no_changes);
                    noMediaChanges = true;
                }
                if (ret.equals("sanityFailed")) {
                    mediaError = AnkiDroidApp.getAppResources().getString(R.string.sync_media_sanity_failed);
                } else {
                    publishProgress(R.string.sync_media_success);
                }
            }
        } catch (RuntimeException e) {
            AnkiDroidApp.saveExceptionReportFile(e, "doInBackgroundSync-mediaSync");
            mediaError = e.getLocalizedMessage();
        }
    }
    if (noChanges && noMediaChanges) {
        data.success = false;
        data.result = new Object[] { "noChanges" };
        return data;
    } else {
        data.success = true;
        TreeSet<Object[]> decks = col.getSched().deckDueTree();
        int[] counts = new int[] { 0, 0, 0 };
        for (Object[] deck : decks) {
            if (((String[]) deck[0]).length == 1) {
                counts[0] += (Integer) deck[2];
                counts[1] += (Integer) deck[3];
                counts[2] += (Integer) deck[4];
            }
        }
        Object[] dc = col.getSched().deckCounts();
        data.result = dc[0];
        data.data = new Object[] { conflictResolution, col, dc[1], dc[2], mediaError };
        return data;
    }
}