Examples with CBZip2InputStream org.apache.hadoop.io.compress.bzip2.CBZip2InputStream used on opensource projects

Example 6 with CBZip2InputStream

use of org.apache.hadoop.io.compress.bzip2.CBZip2InputStream in project ant by apache.

the class BUnzip2 method extract.

/**
 * Do the unbzipping.
 */
protected void extract() {
    if (srcResource.getLastModified() > dest.lastModified()) {
        log("Expanding " + srcResource.getName() + " to " + dest.getAbsolutePath());
        OutputStream out = null;
        CBZip2InputStream zIn = null;
        InputStream fis = null;
        BufferedInputStream bis = null;
        try {
            out = Files.newOutputStream(dest.toPath());
            fis = srcResource.getInputStream();
            bis = new BufferedInputStream(fis);
            int b = bis.read();
            if (b != 'B') {
                throw new BuildException("Invalid bz2 file.", getLocation());
            }
            b = bis.read();
            if (b != 'Z') {
                throw new BuildException("Invalid bz2 file.", getLocation());
            }
            zIn = new CBZip2InputStream(bis, true);
            byte[] buffer = new byte[BUFFER_SIZE];
            int count = 0;
            do {
                out.write(buffer, 0, count);
                count = zIn.read(buffer, 0, buffer.length);
            } while (count != -1);
        } catch (IOException ioe) {
            String msg = "Problem expanding bzip2 " + ioe.getMessage();
            throw new BuildException(msg, ioe, getLocation());
        } finally {
            FileUtils.close(bis);
            FileUtils.close(fis);
            FileUtils.close(out);
            FileUtils.close(zIn);
        }
    }
}

Example 7 with CBZip2InputStream

use of org.apache.hadoop.io.compress.bzip2.CBZip2InputStream in project OsmAnd-tools by osmandapp.

the class IndexCreator method extractOsmToNodesDB.

private OsmDbCreator extractOsmToNodesDB(OsmDbAccessor accessor, File readFile, IProgress progress, IOsmStorageFilter addFilter, int additionId, int shiftId, boolean ovewriteIds, boolean generateNewIds, boolean createTables, OsmDbCreator previous) throws IOException, SQLException, XmlPullParserException {
    boolean pbfFile = false;
    InputStream stream = new BufferedInputStream(new FileInputStream(readFile), 8192 * 4);
    InputStream streamFile = stream;
    long st = System.currentTimeMillis();
    if (readFile.getName().endsWith(".bz2")) {
        // $NON-NLS-1$
        if (stream.read() != 'B' || stream.read() != 'Z') {
        // throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$
        } else {
            stream = new CBZip2InputStream(stream);
        }
    } else if (readFile.getName().endsWith(".gz")) {
        // $NON-NLS-1$
        stream = new GZIPInputStream(stream);
    } else if (readFile.getName().endsWith(".pbf")) {
        // $NON-NLS-1$
        pbfFile = true;
    }
    OsmBaseStorage storage = pbfFile ? new OsmBaseStoragePbf() : new OsmBaseStorage();
    storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
    if (addFilter != null) {
        storage.getFilters().add(addFilter);
    }
    storage.getFilters().add(new IOsmStorageFilter() {

        @Override
        public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity entity) {
            if (indexAddressCreator != null) {
                indexAddressCreator.registerCityIfNeeded(entity);
            }
            // accept to allow db creator parse it
            return true;
        }
    });
    // 1. Loading osm file
    OsmDbCreator dbCreator = new OsmDbCreator(additionId, shiftId, ovewriteIds, generateNewIds);
    if (previous != null) {
        dbCreator.setNodeIds(previous.getNodeIds());
        dbCreator.setWayIds(previous.getWayIds());
        dbCreator.setRelationIds(previous.getRelationIds());
    }
    dbCreator.setBackwardCompatibleIds(backwardCompatibleIds);
    try {
        // $NON-NLS-1$
        setGeneralProgress(progress, "[15 / 100]");
        // $NON-NLS-1$
        progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1);
        // 1 init database to store temporary data
        dbCreator.initDatabase(osmDBdialect, accessor.getDbConn(), createTables);
        storage.getFilters().add(dbCreator);
        if (pbfFile) {
            ((OsmBaseStoragePbf) storage).parseOSMPbf(stream, progress, false);
        } else {
            storage.parseOSM(stream, progress, streamFile, false);
        }
        dbCreator.finishLoading();
        osmDBdialect.commitDatabase(accessor.getDbConn());
        if (log.isInfoEnabled()) {
            // $NON-NLS-1$
            log.info("File parsed : " + (System.currentTimeMillis() - st));
        }
        progress.finishTask();
        return dbCreator;
    } finally {
        if (log.isInfoEnabled()) {
            // $NON-NLS-1$
            log.info("File indexed : " + (System.currentTimeMillis() - st));
        }
    }
}

Example 8 with CBZip2InputStream

use of org.apache.hadoop.io.compress.bzip2.CBZip2InputStream in project OsmAnd-tools by osmandapp.

the class FixBasemapRoads method parseOsmFile.

private OsmBaseStorage parseOsmFile(File read) throws FileNotFoundException, IOException, XmlPullParserException {
    OsmBaseStorage storage = new OsmBaseStorage();
    InputStream stream = new BufferedInputStream(new FileInputStream(read), 8192 * 4);
    InputStream streamFile = stream;
    if (read.getName().endsWith(".bz2")) {
        // $NON-NLS-1$
        if (stream.read() != 'B' || stream.read() != 'Z') {
        // throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$
        } else {
            stream = new CBZip2InputStream(stream);
        }
    } else if (read.getName().endsWith(".gz")) {
        // $NON-NLS-1$
        stream = new GZIPInputStream(stream);
    }
    storage.parseOSM(stream, new ConsoleProgressImplementation(), streamFile, true);
    return storage;
}

Example 9 with CBZip2InputStream

use of org.apache.hadoop.io.compress.bzip2.CBZip2InputStream in project OsmAnd-tools by osmandapp.

the class WikiDatabasePreparation method processWikipedia.

protected static void processWikipedia(final String wikiPg, String lang, Map<Long, LatLon> links, String sqliteFileName) throws ParserConfigurationException, SAXException, FileNotFoundException, IOException, SQLException, ComponentLookupException {
    SAXParser sx = SAXParserFactory.newInstance().newSAXParser();
    InputStream streamFile = new BufferedInputStream(new FileInputStream(wikiPg), 8192 * 4);
    InputStream stream = streamFile;
    if (stream.read() != 'B' || stream.read() != 'Z') {
        throw new RuntimeException(// $NON-NLS-1$
        "The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
    }
    CBZip2InputStream zis = new CBZip2InputStream(stream);
    Reader reader = new InputStreamReader(zis, "UTF-8");
    InputSource is = new InputSource(reader);
    is.setEncoding("UTF-8");
    final WikiOsmHandler handler = new WikiOsmHandler(sx, streamFile, lang, links, new File(sqliteFileName));
    sx.parse(is, handler);
    handler.finish();
}

Example 10 with CBZip2InputStream

use of org.apache.hadoop.io.compress.bzip2.CBZip2InputStream in project OsmAnd-tools by osmandapp.

the class WikiVoyagePreparation method processWikivoyage.

protected static void processWikivoyage(final String wikiPg, String lang, String sqliteFileName, File langlinkConn) throws ParserConfigurationException, SAXException, FileNotFoundException, IOException, SQLException, ComponentLookupException {
    SAXParser sx = SAXParserFactory.newInstance().newSAXParser();
    InputStream streamFile = new BufferedInputStream(new FileInputStream(wikiPg), 8192 * 4);
    InputStream stream = streamFile;
    if (stream.read() != 'B' || stream.read() != 'Z') {
        stream.close();
        throw new RuntimeException(// $NON-NLS-1$
        "The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
    }
    CBZip2InputStream zis = new CBZip2InputStream(stream);
    Reader reader = new InputStreamReader(zis, "UTF-8");
    InputSource is = new InputSource(reader);
    is.setEncoding("UTF-8");
    final WikiOsmHandler handler = new WikiOsmHandler(sx, streamFile, lang, new File(sqliteFileName), langlinkConn);
    sx.parse(is, handler);
    handler.finish();
}