Examples with IngestResults org.locationtech.geogig.remote.BinaryPackedObjects.IngestResults used on opensource projects

Example 1 with IngestResults

use of org.locationtech.geogig.remote.BinaryPackedObjects.IngestResults in project GeoGig by boundlessgeo.

the class HttpRemoteRepo method fetchMoreData.

/**
     * Retrieve objects from the remote repository, and update have/want lists accordingly.
     * Specifically, any retrieved commits are removed from the want list and added to the have
     * list, and any parents of those commits are removed from the have list (it only represents the
     * most recent common commits.) Retrieved objects are added to the local repository, and the
     * want/have lists are updated in-place.
     * 
     * @param want a list of ObjectIds that need to be fetched
     * @param have a list of ObjectIds that are in common with the remote repository
     * @param progress
     */
private void fetchMoreData(final List<ObjectId> want, final Set<ObjectId> have, final ProgressListener progress) {
    final JsonObject message = createFetchMessage(want, have);
    final URL resourceURL;
    try {
        resourceURL = new URL(repositoryURL.toString() + "/repo/batchobjects");
    } catch (MalformedURLException e) {
        throw Throwables.propagate(e);
    }
    final HttpURLConnection connection;
    try {
        final Gson gson = new Gson();
        OutputStream out;
        final Writer writer;
        connection = (HttpURLConnection) resourceURL.openConnection();
        connection.setDoOutput(true);
        connection.setDoInput(true);
        connection.addRequestProperty("Accept-Encoding", "gzip");
        out = connection.getOutputStream();
        writer = new OutputStreamWriter(out);
        gson.toJson(message, writer);
        writer.flush();
        out.flush();
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
    final HttpUtils.ReportingInputStream in = HttpUtils.getResponseStream(connection);
    BinaryPackedObjects unpacker = new BinaryPackedObjects(localRepository.objectDatabase());
    BinaryPackedObjects.Callback callback = new BinaryPackedObjects.Callback() {

        @Override
        public void callback(Supplier<RevObject> supplier) {
            RevObject object = supplier.get();
            progress.setProgress(progress.getProgress() + 1);
            if (object instanceof RevCommit) {
                RevCommit commit = (RevCommit) object;
                want.remove(commit.getId());
                have.removeAll(commit.getParentIds());
                have.add(commit.getId());
            } else if (object instanceof RevTag) {
                RevTag tag = (RevTag) object;
                want.remove(tag.getId());
                have.remove(tag.getCommitId());
                have.add(tag.getId());
            }
        }
    };
    Stopwatch sw = Stopwatch.createStarted();
    IngestResults ingestResults = unpacker.ingest(in, callback);
    sw.stop();
    String msg = String.format("Processed %,d objects. Inserted: %,d. Existing: %,d. Time: %s. Compressed size: %,d bytes. Uncompressed size: %,d bytes.", ingestResults.total(), ingestResults.getInserted(), ingestResults.getExisting(), sw, in.compressedSize(), in.unCompressedSize());
    LOGGER.info(msg);
    progress.setDescription(msg);
}

Example 2 with IngestResults

use of org.locationtech.geogig.remote.BinaryPackedObjects.IngestResults in project GeoGig by boundlessgeo.

the class SendObjectResource method post.

@Override
public void post(Representation entity) {
    InputStream input = null;
    Request request = getRequest();
    try {
        LOGGER.info("Receiving objects from {}", request.getClientInfo().getAddress());
        Representation representation = request.getEntity();
        input = representation.getStream();
        final GeoGIG ggit = getGeogig(request).get();
        final BinaryPackedObjects unpacker = new BinaryPackedObjects(ggit.getRepository().objectDatabase());
        CountingInputStream countingStream = new CountingInputStream(input);
        Stopwatch sw = Stopwatch.createStarted();
        IngestResults ingestResults = unpacker.ingest(countingStream);
        sw.stop();
        LOGGER.info(String.format("SendObjectResource: Processed %,d objects.\nInserted: %,d.\nExisting: %,d.\nTime to process: %s.\nStream size: %,d bytes.\n", ingestResults.total(), ingestResults.getInserted(), ingestResults.getExisting(), sw, countingStream.getCount()));
    } catch (IOException e) {
        LOGGER.warn("Error processing incoming objects from {}", request.getClientInfo().getAddress(), e);
        throw new RestletException(e.getMessage(), Status.SERVER_ERROR_INTERNAL, e);
    } finally {
        if (input != null)
            Closeables.closeQuietly(input);
    }
}