Examples with BinaryContent ddf.catalog.data.BinaryContent used on opensource projects

Example 81 with BinaryContent

use of ddf.catalog.data.BinaryContent in project ddf by codice.

the class TestAtomTransformer method testNoCreatedDate.

@Test
public void testNoCreatedDate() throws IOException, CatalogTransformerException, XpathException, SAXException {
    // given
    AtomTransformer transformer = new AtomTransformer();
    MetacardTransformer metacardTransformer = getXmlMetacardTransformerStub();
    transformer.setMetacardTransformer(metacardTransformer);
    setDefaultSystemConfiguration();
    SourceResponse response = mock(SourceResponse.class);
    when(response.getHits()).thenReturn(new Long(1));
    when(response.getRequest()).thenReturn(getStubRequest());
    ResultImpl result1 = new ResultImpl();
    MetacardStub metacard = new MetacardStub("");
    metacard.setId(SAMPLE_ID);
    metacard.setCreatedDate(null);
    result1.setMetacard(metacard);
    when(response.getResults()).thenReturn(Arrays.asList((Result) result1));
    result1.setRelevanceScore(0.3345);
    // when
    BinaryContent binaryContent = transformer.transform(response, null);
    // then
    assertThat(binaryContent.getMimeType(), is(AtomTransformer.MIME_TYPE));
    byte[] bytes = binaryContent.getByteArray();
    String output = new String(bytes);
    assertFeedCompliant(output);
    assertEntryCompliant(output);
    validateAgainstAtomSchema(bytes);
    assertXpathNotExists("atom:feed/atom:entry/atom:published", output);
}

Example 82 with BinaryContent

use of ddf.catalog.data.BinaryContent in project ddf by codice.

the class TestAtomTransformer method testNoModifiedDate.

@Test
public void testNoModifiedDate() throws IOException, CatalogTransformerException, XpathException, SAXException {
    // given
    AtomTransformer transformer = new AtomTransformer();
    MetacardTransformer metacardTransformer = getXmlMetacardTransformerStub();
    transformer.setMetacardTransformer(metacardTransformer);
    setDefaultSystemConfiguration();
    SourceResponse response = mock(SourceResponse.class);
    when(response.getHits()).thenReturn(new Long(1));
    when(response.getRequest()).thenReturn(getStubRequest());
    ResultImpl result1 = new ResultImpl();
    MetacardStub metacard = new MetacardStub("");
    metacard.setId(SAMPLE_ID);
    metacard.setModifiedDate(null);
    result1.setMetacard(metacard);
    when(response.getResults()).thenReturn(Arrays.asList((Result) result1));
    result1.setRelevanceScore(0.3345);
    // when
    BinaryContent binaryContent = transformer.transform(response, null);
    // then
    assertThat(binaryContent.getMimeType(), is(AtomTransformer.MIME_TYPE));
    byte[] bytes = binaryContent.getByteArray();
    String output = new String(bytes);
    assertFeedCompliant(output);
    assertEntryCompliant(output);
    validateAgainstAtomSchema(bytes);
    assertXpathExists("atom:feed/atom:entry/atom:updated", output);
}

Example 83 with BinaryContent

use of ddf.catalog.data.BinaryContent in project ddf by codice.

the class TestAtomTransformer method testItemsPerPageNegativeInteger.

@Test
public void testItemsPerPageNegativeInteger() throws IOException, CatalogTransformerException, XpathException, SAXException {
    // given
    AtomTransformer transformer = new AtomTransformer();
    MetacardTransformer metacardTransformer = getXmlMetacardTransformerStub();
    transformer.setMetacardTransformer(metacardTransformer);
    setDefaultSystemConfiguration();
    SourceResponse response = mock(SourceResponse.class);
    when(response.getHits()).thenReturn(new Long(1));
    QueryImpl query = new QueryImpl(FILTER_BUILDER.attribute(Metacard.METADATA).text("you"));
    query.setPageSize(-1);
    query.setStartIndex(2);
    query.setRequestsTotalResultsCount(true);
    QueryRequestImpl queryRequestImpl = new QueryRequestImpl(query);
    when(response.getRequest()).thenReturn(queryRequestImpl);
    ResultImpl result1 = new ResultImpl();
    MetacardStub metacard = new MetacardStub("");
    metacard.setId(SAMPLE_ID);
    result1.setMetacard(metacard);
    when(response.getResults()).thenReturn(Arrays.asList((Result) result1));
    // when
    BinaryContent binaryContent = transformer.transform(response, null);
    // then
    assertThat(binaryContent.getMimeType(), is(AtomTransformer.MIME_TYPE));
    byte[] bytes = binaryContent.getByteArray();
    String output = new String(bytes);
    assertFeedCompliant(output);
    assertEntryCompliant(output);
    validateAgainstAtomSchema(bytes);
    assertXpathEvaluatesTo("1", "/atom:feed/os:itemsPerPage", output);
}

Example 84 with BinaryContent

use of ddf.catalog.data.BinaryContent in project ddf by codice.

the class ExportCommand method writeToZip.

private void writeToZip(/*Mutable,IO*/
ZipFile zipFile, Result result) {
    ZipParameters parameters = new ZipParameters();
    parameters.setSourceExternalStream(true);
    String id = result.getMetacard().getId();
    parameters.setFileNameInZip(Paths.get("metacards", id.substring(0, 3), id, "metacard", id + ".xml").toString());
    try {
        BinaryContent binaryMetacard = transformer.transform(result.getMetacard(), Collections.emptyMap());
        zipFile.addStream(binaryMetacard.getInputStream(), parameters);
    } catch (ZipException e) {
        LOGGER.error("Error processing result and adding to ZIP", e);
        throw new CatalogCommandRuntimeException(e);
    } catch (CatalogTransformerException e) {
        LOGGER.warn("Could not transform metacard. Metacard will not be added to zip [{}]", result.getMetacard().getId());
        console.printf("%sCould not transform metacard. Metacard will not be included in export. %s - %s%s%n", Ansi.ansi().fg(Ansi.Color.RED).toString(), result.getMetacard().getId(), result.getMetacard().getTitle(), Ansi.ansi().reset().toString());
    }
}

Example 85 with BinaryContent

use of ddf.catalog.data.BinaryContent in project ddf by codice.

the class DumpCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    if (FilenameUtils.getExtension(dirPath).equals("") && !dirPath.endsWith(File.separator)) {
        dirPath += File.separator;
    }
    final File dumpDir = new File(dirPath);
    if (!dumpDir.exists()) {
        printErrorMessage("Directory [" + dirPath + "] must exist.");
        console.println("If the directory does indeed exist, try putting the path in quotes.");
        return null;
    }
    if (!dumpDir.isDirectory()) {
        printErrorMessage("Path [" + dirPath + "] must be a directory.");
        return null;
    }
    if (!SERIALIZED_OBJECT_ID.matches(transformerId)) {
        transformers = getTransformers();
        if (transformers == null) {
            console.println(transformerId + " is an invalid metacard transformer.");
            return null;
        }
    }
    if (StringUtils.isNotBlank(zipFileName) && new File(dirPath + zipFileName).exists()) {
        console.println("Cannot dump Catalog.  Zip file " + zipFileName + " already exists.");
        return null;
    }
    SecurityLogger.audit("Called catalog:dump command with path : {}", dirPath);
    CatalogFacade catalog = getCatalog();
    if (StringUtils.isNotBlank(zipFileName)) {
        zipArgs = new HashMap<>();
        zipArgs.put(FILE_PATH, dirPath + zipFileName);
    }
    QueryImpl query = new QueryImpl(getFilter());
    query.setRequestsTotalResultsCount(false);
    query.setPageSize(pageSize);
    Map<String, Serializable> props = new HashMap<>();
    // Avoid caching all results while dumping with native query mode
    props.put("mode", "native");
    final AtomicLong resultCount = new AtomicLong(0);
    long start = System.currentTimeMillis();
    SourceResponse response = catalog.query(new QueryRequestImpl(query, props));
    BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded);
    RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
    final ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, rejectedExecutionHandler);
    while (response.getResults().size() > 0) {
        response = catalog.query(new QueryRequestImpl(query, props));
        if (StringUtils.isNotBlank(zipFileName)) {
            try {
                Optional<QueryResponseTransformer> zipCompression = getZipCompression();
                if (zipCompression.isPresent()) {
                    BinaryContent binaryContent = zipCompression.get().transform(response, zipArgs);
                    if (binaryContent != null) {
                        IOUtils.closeQuietly(binaryContent.getInputStream());
                    }
                    Long resultSize = (long) response.getResults().size();
                    printStatus(resultCount.addAndGet(resultSize));
                }
            } catch (InvalidSyntaxException e) {
                LOGGER.info("No Zip Transformer found.  Unable export metacards to a zip file.");
            }
        } else if (multithreaded > 1) {
            final List<Result> results = new ArrayList<>(response.getResults());
            executorService.submit(() -> {
                boolean transformationFailed = false;
                for (final Result result : results) {
                    Metacard metacard = result.getMetacard();
                    try {
                        exportMetacard(dumpDir, metacard);
                    } catch (IOException | CatalogTransformerException e) {
                        transformationFailed = true;
                        LOGGER.debug("Failed to dump metacard {}", metacard.getId(), e);
                        executorService.shutdownNow();
                    }
                    printStatus(resultCount.incrementAndGet());
                }
                if (transformationFailed) {
                    LOGGER.info("One or more metacards failed to transform. Enable debug log for more details.");
                }
            });
        } else {
            for (final Result result : response.getResults()) {
                Metacard metacard = result.getMetacard();
                exportMetacard(dumpDir, metacard);
                printStatus(resultCount.incrementAndGet());
            }
        }
        if (response.getResults().size() < pageSize || pageSize == -1) {
            break;
        }
        if (pageSize > 0) {
            query.setStartIndex(query.getStartIndex() + pageSize);
        }
    }
    executorService.shutdown();
    while (!executorService.isTerminated()) {
        try {
            TimeUnit.MILLISECONDS.sleep(100);
        } catch (InterruptedException e) {
        // ignore
        }
    }
    long end = System.currentTimeMillis();
    String elapsedTime = timeFormatter.print(new Period(start, end).withMillis(0));
    console.printf(" %d file(s) dumped in %s\t%n", resultCount.get(), elapsedTime);
    LOGGER.debug("{} file(s) dumped in {}", resultCount.get(), elapsedTime);
    console.println();
    SecurityLogger.audit("Exported {} files to {}", resultCount.get(), dirPath);
    return null;
}