Examples with CountingOutputStream com.google.common.io.CountingOutputStream used on opensource projects

Example 26 with CountingOutputStream

use of com.google.common.io.CountingOutputStream in project beam by apache.

the class PackageUtil method createPackageAttributes.

/**
   * Compute and cache the attributes of a classpath element that we will need to stage it.
   *
   * @param source the file or directory to be staged.
   * @param stagingPath The base location for staged classpath elements.
   * @param overridePackageName If non-null, use the given value as the package name
   *                            instead of generating one automatically.
   * @return a {@link PackageAttributes} that containing metadata about the object to be staged.
   */
static PackageAttributes createPackageAttributes(File source, String stagingPath, @Nullable String overridePackageName) {
    boolean directory = source.isDirectory();
    // Compute size and hash in one pass over file or directory.
    Hasher hasher = Hashing.md5().newHasher();
    OutputStream hashStream = Funnels.asOutputStream(hasher);
    try (CountingOutputStream countingOutputStream = new CountingOutputStream(hashStream)) {
        if (!directory) {
            // Files are staged as-is.
            Files.asByteSource(source).copyTo(countingOutputStream);
        } else {
            // Directories are recursively zipped.
            ZipFiles.zipDirectory(source, countingOutputStream);
        }
        countingOutputStream.flush();
        long size = countingOutputStream.getCount();
        String hash = Base64Variants.MODIFIED_FOR_URL.encode(hasher.hash().asBytes());
        // Create the DataflowPackage with staging name and location.
        String uniqueName = getUniqueContentName(source, hash);
        String resourcePath = FileSystems.matchNewResource(stagingPath, true).resolve(uniqueName, StandardResolveOptions.RESOLVE_FILE).toString();
        DataflowPackage target = new DataflowPackage();
        target.setName(overridePackageName != null ? overridePackageName : uniqueName);
        target.setLocation(resourcePath);
        return new PackageAttributes(size, hash, directory, target, source.getPath());
    } catch (IOException e) {
        throw new RuntimeException("Package setup failure for " + source, e);
    }
}

Example 27 with CountingOutputStream

use of com.google.common.io.CountingOutputStream in project ddf by codice.

the class ReliableResourceDownloaderTest method testClientOutputStreamException.

@Test
@Ignore
public // Can't figure out how to throw IOExcetion from CountingOutputStream
void testClientOutputStreamException() throws Exception {
    downloaderConfig.setCacheEnabled(true);
    ResourceCacheImpl mockCache = mock(ResourceCacheImpl.class);
    when(mockCache.isPending(anyString())).thenReturn(false);
    when(mockCache.getProductCacheDirectory()).thenReturn(productCacheDirectory);
    downloaderConfig.setResourceCache(mockCache);
    mis = new MockInputStream(productInputFilename);
    ResourceResponse mockResponse = getMockResourceResponse(mis);
    ReliableResourceDownloader downloader = new ReliableResourceDownloader(downloaderConfig, new AtomicBoolean(), "123", mockResponse, getMockRetriever());
    downloader.setupDownload(mockMetacard, new DownloadStatusInfoImpl());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    CountingOutputStream mockCountingFbos = new CountingOutputStream(baos);
    IOUtils.closeQuietly(baos);
    downloader.setCountingOutputStream(mockCountingFbos);
    downloader.run();
    verify(mockPublisher, times(1)).postRetrievalStatus(any(ResourceResponse.class), eq(ProductRetrievalStatus.CANCELLED), any(Metacard.class), anyString(), anyLong(), eq(DOWNLOAD_ID));
    verify(mockCache, times(1)).removePendingCacheEntry(anyString());
    assertThat(downloaderConfig.isCacheEnabled(), is(false));
}

Example 28 with CountingOutputStream

use of com.google.common.io.CountingOutputStream in project ddf by codice.

the class ReliableResourceDownloader method setupDownload.

public ResourceResponse setupDownload(Metacard metacard, DownloadStatusInfo downloadStatusInfo) {
    Resource resource = resourceResponse.getResource();
    MimeType mimeType = resource.getMimeType();
    String resourceName = resource.getName();
    fbos = new FileBackedOutputStream(DEFAULT_FILE_BACKED_OUTPUT_STREAM_THRESHOLD);
    countingFbos = new CountingOutputStream(fbos);
    streamReadByClient = new ReliableResourceInputStream(fbos, countingFbos, downloadState, downloadIdentifier, resourceResponse);
    this.metacard = metacard;
    // Create new ResourceResponse to return that will encapsulate the
    // ReliableResourceInputStream that will be read by the client simultaneously as the product
    // is cached to disk (if caching is enabled)
    ResourceImpl newResource = new ResourceImpl(streamReadByClient, mimeType, resourceName);
    resourceResponse = new ResourceResponseImpl(resourceResponse.getRequest(), resourceResponse.getProperties(), newResource);
    // Get handle to retrieved product's InputStream
    resourceInputStream = resource.getInputStream();
    eventListener.setDownloadMap(downloadIdentifier, resourceResponse);
    downloadStatusInfo.addDownloadInfo(downloadIdentifier, this, resourceResponse);
    if (downloaderConfig.isCacheEnabled()) {
        CacheKey keyMaker = null;
        String key = null;
        try {
            keyMaker = new CacheKey(metacard, resourceResponse.getRequest());
            key = keyMaker.generateKey();
        } catch (IllegalArgumentException e) {
            LOGGER.info("Cannot create cache key for resource with metacard ID = {}", metacard.getId());
            return resourceResponse;
        }
        if (!resourceCache.isPending(key)) {
            // Fully qualified path to cache file that will be written to.
            // Example:
            // <INSTALL-DIR>/data/product-cache/<source-id>-<metacard-id>
            // <INSTALL-DIR>/data/product-cache/ddf.distribution-abc123
            filePath = FilenameUtils.concat(resourceCache.getProductCacheDirectory(), key);
            if (filePath == null) {
                LOGGER.info("Unable to create cache for cache directory {} and key {} - no caching will be done.", resourceCache.getProductCacheDirectory(), key);
                return resourceResponse;
            }
            reliableResource = new ReliableResource(key, filePath, mimeType, resourceName, metacard);
            resourceCache.addPendingCacheEntry(reliableResource);
            try {
                fos = FileUtils.openOutputStream(new File(filePath));
                doCaching = true;
                this.downloadState.setCacheEnabled(true);
            } catch (IOException e) {
                LOGGER.info("Unable to open cache file {} - no caching will be done.", filePath);
            }
        } else {
            LOGGER.debug("Cache key {} is already pending caching", key);
        }
    }
    return resourceResponse;
}

Example 29 with CountingOutputStream

use of com.google.common.io.CountingOutputStream in project GeoGig by boundlessgeo.

the class BinaryPackedChanges method write.

/**
     * Writes the set of changes to the provided output stream.
     * 
     * @param out the stream to write to
     * @param changes the changes to write
     * @throws IOException
     * @return the number of objects written
     */
public long write(OutputStream out, Iterator<DiffEntry> changes) throws IOException {
    final ObjectDatabase objectDatabase = repository.objectDatabase();
    out = new CountingOutputStream(out);
    // avoids sending the same metadata object multiple times
    Set<ObjectId> writtenMetadataIds = new HashSet<ObjectId>();
    // buffer to avoid ObjectId cloning its internal state for each object
    byte[] oidbuffer = new byte[ObjectId.NUM_BYTES];
    long objectCount = 0;
    while (changes.hasNext()) {
        DiffEntry diff = changes.next();
        if (diff.isDelete()) {
            out.write(CHUNK_TYPE.DIFF_ENTRY.value());
        } else {
            // its a change or an addition, new object is guaranteed to be present
            NodeRef newObject = diff.getNewObject();
            ObjectId metadataId = newObject.getMetadataId();
            if (writtenMetadataIds.contains(metadataId)) {
                out.write(CHUNK_TYPE.OBJECT_AND_DIFF_ENTRY.value());
            } else {
                out.write(CHUNK_TYPE.METADATA_OBJECT_AND_DIFF_ENTRY.value());
                RevObject metadata = objectDatabase.get(metadataId);
                writeObjectId(metadataId, out, oidbuffer);
                serializer.createObjectWriter(metadata.getType()).write(metadata, out);
                writtenMetadataIds.add(metadataId);
                objectCount++;
            }
            ObjectId objectId = newObject.objectId();
            writeObjectId(objectId, out, oidbuffer);
            RevObject object = objectDatabase.get(objectId);
            serializer.createObjectWriter(object.getType()).write(object, out);
            objectCount++;
        }
        DataOutput dataOut = new DataOutputStream(out);
        FormatCommonV1.writeDiff(diff, dataOut);
    }
    // signal the end of changes
    out.write(CHUNK_TYPE.FILTER_FLAG.value());
    final boolean filtersApplied = changes instanceof FilteredDiffIterator && ((FilteredDiffIterator) changes).wasFiltered();
    out.write(filtersApplied ? 1 : 0);
    LOGGER.info(String.format("Written %,d bytes to remote accounting for %,d objects.", ((CountingOutputStream) out).getCount(), objectCount));
    return objectCount;
}

Example 30 with CountingOutputStream

use of com.google.common.io.CountingOutputStream in project oap by oaplatform.

the class Writer method write.

public synchronized void write(byte[] buffer, int offset, int length, Consumer<String> error) throws LoggerException {
    try {
        refresh();
        Path filename = filename();
        if (out == null)
            if (Files.isFileEncodingValid(filename))
                out = new CountingOutputStream(IoStreams.out(filename, Encoding.from(ext), bufferSize, true));
            else {
                error.accept("corrupted file, cannot append " + filename);
                log.error("corrupted file, cannot append {}", filename);
                Files.rename(filename, logDirectory.resolve(".corrupted").resolve(logDirectory.relativize(filename)));
                out = new CountingOutputStream(IoStreams.out(filename, Encoding.from(ext), bufferSize));
            }
        log.trace("writing {} bytes to {}", length, this);
        out.write(buffer, offset, length);
    } catch (IOException e) {
        log.error(e.getMessage(), e);
        try {
            closeOutput();
        } finally {
            out = null;
        }
        throw new LoggerException(e);
    }
}