Examples with ByteSource com.google.common.io.ByteSource used on opensource projects

Example 51 with ByteSource

use of com.google.common.io.ByteSource in project ddf by codice.

the class CatalogFrameworkImplTest method testCreateStorageWithAttributeOverrides.

/**
     * Tests that the framework properly passes a create request to the local provider with attribute overrides.
     */
@Test
public void testCreateStorageWithAttributeOverrides() throws Exception {
    List<ContentItem> contentItems = new ArrayList<>();
    Map<String, Serializable> propertiesMap = new HashMap<>();
    HashMap<String, String> attributeMap = new HashMap<>();
    attributeMap.put(Metacard.TITLE, "test");
    attributeMap.put("foo", "bar");
    propertiesMap.put(Constants.ATTRIBUTE_OVERRIDES_KEY, attributeMap);
    MetacardImpl newCard = new MetacardImpl();
    newCard.setId(null);
    MetacardType metacardType = mock(MetacardType.class);
    AttributeDescriptor stringAttributeDescriptor = new AttributeDescriptorImpl(Metacard.TITLE, true, true, true, true, new AttributeType<String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Class<String> getBinding() {
            return String.class;
        }

        @Override
        public AttributeFormat getAttributeFormat() {
            return AttributeFormat.STRING;
        }
    });
    when(metacardType.getAttributeDescriptor(Metacard.TITLE)).thenReturn(stringAttributeDescriptor);
    newCard.setType(metacardType);
    ByteSource byteSource = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return new ByteArrayInputStream("blah".getBytes());
        }
    };
    ContentItemImpl newItem = new ContentItemImpl(uuidGenerator.generateUuid(), byteSource, "application/octet-stream", "blah", 0L, newCard);
    contentItems.add(newItem);
    CreateResponse response = framework.create(new CreateStorageRequestImpl(contentItems, propertiesMap));
    assertEquals(response.getCreatedMetacards().size(), provider.size());
    assertEquals(response.getCreatedMetacards().size(), storageProvider.size());
    for (Metacard curCard : response.getCreatedMetacards()) {
        assertNotNull(curCard.getId());
        // Assert valid attribute is set for the metacard
        assertThat(curCard.getTitle(), is("test"));
        // Assert invalid attribute is not set for the metacard
        assertThat(curCard.getAttribute("foo"), nullValue());
    }
    // Assert That Attribute Overrides do not exist after create
    assertThat(attributeMap.get(Constants.ATTRIBUTE_OVERRIDES_KEY), nullValue());
}

Example 52 with ByteSource

use of com.google.common.io.ByteSource in project ddf by codice.

the class CatalogFrameworkImplTest method testCreateStorage.

/**
     * Tests that the framework properly passes a create request to the local provider.
     */
@Test
public void testCreateStorage() throws Exception {
    List<ContentItem> contentItems = new ArrayList<>();
    MetacardImpl newCard = new MetacardImpl();
    newCard.setId(null);
    ByteSource byteSource = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return new ByteArrayInputStream("blah".getBytes());
        }
    };
    ContentItemImpl newItem = new ContentItemImpl(uuidGenerator.generateUuid(), byteSource, "application/octet-stream", "blah", 0L, newCard);
    contentItems.add(newItem);
    CreateResponse response = framework.create(new CreateStorageRequestImpl(contentItems, null));
    assertEquals(response.getCreatedMetacards().size(), provider.size());
    assertEquals(response.getCreatedMetacards().size(), storageProvider.size());
    for (Metacard curCard : response.getCreatedMetacards()) {
        assertNotNull(curCard.getId());
    }
    // make sure that the event was posted correctly
    assertTrue(eventAdmin.wasEventPosted());
    Metacard[] array = {};
    array = response.getCreatedMetacards().toArray(array);
    assertTrue(eventAdmin.wasEventPosted());
    assertEquals(eventAdmin.getLastEvent(), array[array.length - 1]);
}

Example 53 with ByteSource

use of com.google.common.io.ByteSource in project ddf by codice.

the class GeoNamesFileExtractor method unZipInputStream.

/**
     * Unzips a file and returns the output as a new InputStream
     *
     * @param resource    - the name of the resource file to be unzipped
     * @param inputStream - the InputStream for the file to be unzipped
     * @return - the unzipped file as an InputStream
     * @throws GeoEntryExtractionException when the given file fails to be unzipped.
     */
private InputStream unZipInputStream(String resource, InputStream inputStream) throws GeoEntryExtractionException {
    try (TemporaryFileBackedOutputStream bufferedOutputStream = new TemporaryFileBackedOutputStream(BUFFER_SIZE);
        ZipInputStream zipInputStream = new ZipInputStream(inputStream)) {
        ZipEntry zipEntry;
        while ((zipEntry = zipInputStream.getNextEntry()) != null) {
            // GeoNames <filename>.zip files will contain <filename>.txt and readme.txt
            if (!zipEntry.getName().equals("readme.txt")) {
                byte[] data = new byte[BUFFER_SIZE];
                int bytesRead;
                while ((bytesRead = zipInputStream.read(data, 0, BUFFER_SIZE)) != -1) {
                    bufferedOutputStream.write(data, 0, bytesRead);
                }
                ByteSource zipByteSource = bufferedOutputStream.asByteSource();
                bufferedOutputStream.flush();
                fileSize = zipByteSource.size();
                return zipByteSource.openBufferedStream();
            }
        }
    } catch (IOException e) {
        throw new GeoEntryExtractionException("Unable to unzip " + resource, e);
    }
    throw new GeoEntryExtractionException("Unable to unzip " + resource);
}

Example 54 with ByteSource

use of com.google.common.io.ByteSource in project ddf by codice.

the class GeoNamesFileExtractor method getInputStreamFromUrl.

/**
     * Download a GeoNames .zip file from a remote location
     *
     * @param resource         - the name of the zip file to download ( ex. AD )
     * @param response         - the response from the get request
     * @param inputStream      - the InputStream from the web connection
     * @param progressCallback -  the callback to receive updates about the progress, may be
     *                         null if you don't want any updates
     * @throws GeoNamesRemoteDownloadException when the connection could not be established or the
     *                                         file could not be downloaded.
     */
private InputStream getInputStreamFromUrl(String resource, Response response, InputStream inputStream, ProgressCallback progressCallback) throws GeoNamesRemoteDownloadException {
    int responseCode = 0;
    try (TemporaryFileBackedOutputStream fileOutputStream = new TemporaryFileBackedOutputStream(BUFFER_SIZE)) {
        responseCode = response.getStatus();
        int totalFileSize = response.getLength();
        if (inputStream == null) {
            throw new GeoNamesRemoteDownloadException("Unable to get input stream from " + url + ".  Server responded with : " + responseCode);
        }
        double totalBytesRead = 0.0;
        int bytesRead = -1;
        byte[] buffer = new byte[BUFFER_SIZE];
        while ((bytesRead = inputStream.read(buffer)) != -1) {
            fileOutputStream.write(buffer, 0, bytesRead);
            totalBytesRead += bytesRead;
            if (progressCallback != null) {
                progressCallback.updateProgress((int) ((totalBytesRead / totalFileSize) * 50));
            }
        }
        if (progressCallback != null) {
            progressCallback.updateProgress(50);
        }
        ByteSource byteSource = fileOutputStream.asByteSource();
        fileOutputStream.flush();
        inputStream.close();
        closeConnection();
        return byteSource.openBufferedStream();
    } catch (IOException e) {
        throw new GeoNamesRemoteDownloadException("Unable to download " + resource + " from " + url + ".  Server responded with : " + responseCode, e);
    }
}

Example 55 with ByteSource

use of com.google.common.io.ByteSource in project ddf by codice.

the class IngestCommand method submitToStorageProvider.

private void submitToStorageProvider(List<Metacard> metacardList) {
    metacardList.stream().filter(metacard -> metacardFileMapping.containsKey(metacard.getId())).map(metacard -> {
        List<File> fileList = metacardFileMapping.get(metacard.getId());
        List<ContentItem> contentItemList = new ArrayList<>();
        ContentItem contentItem;
        for (File file : fileList) {
            ByteSource byteSource = com.google.common.io.Files.asByteSource(file);
            String fileName = file.getName().split("-")[1];
            String fragment = null;
            if (!file.getPath().contains(CONTENT + File.separator + metacard.getId())) {
                fragment = StringUtils.substringBetween(file.getPath(), CONTENT + File.separator, File.separator + metacard.getId());
            }
            contentItem = new ContentItemImpl(metacard.getId(), fragment, byteSource, metacard.getContentTypeName(), fileName, file.length(), metacard);
            contentItemList.add(contentItem);
        }
        return new CreateStorageRequestImpl(contentItemList, metacard.getId(), new HashMap<>());
    }).forEach(createStorageRequest -> {
        try {
            storageProvider.create(createStorageRequest);
            storageProvider.commit(createStorageRequest);
        } catch (StorageException e) {
            LOGGER.debug("Unable to create content for {}", createStorageRequest.getId(), e);
            try {
                storageProvider.rollback(createStorageRequest);
            } catch (StorageException e1) {
                LOGGER.debug("Unable to perform rollback on temporary content for {} ", createStorageRequest.getId(), e1);
            }
        }
    });
}