Examples with InputTransformer ddf.catalog.transform.InputTransformer used on opensource projects

Example 41 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project alliance by codice.

the class MpegTsInputTransformerTest method setup.

@Before
public void setup() throws IOException, CatalogTransformerException {
    metacardTypes = Collections.singletonList(mock(MetacardType.class));
    stanag4609Processor = mock(Stanag4609Processor.class);
    klvHandlerFactory = mock(KlvHandlerFactory.class);
    defaultKlvHandler = mock(KlvHandler.class);
    streamParser = mock(Stanag4609TransportStreamParser.class);
    metacard = new MetacardImpl();
    inputTransformer = mock(InputTransformer.class);
    stanagParserFactory = mock(StanagParserFactory.class);
    klvProcessor = mock(KlvProcessor.class);
    when(inputTransformer.transform(any(), any())).thenReturn(metacard);
    when(stanagParserFactory.createParser(any())).thenReturn(() -> {
        try {
            return streamParser.parse();
        } catch (Exception e) {
            throw new Stanag4609ParseException(e);
        }
    });
}

Example 42 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class InputTransformerProducer method generateMetacard.

private Optional<Metacard> generateMetacard(MimeType mimeType, MimeTypeToTransformerMapper mapper, TemporaryFileBackedOutputStream tfbos, String metacardId) {
    LOGGER.trace("ENTERING: generateMetacard");
    List<InputTransformer> listOfCandidates = mapper.findMatches(InputTransformer.class, mimeType);
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("List of matches for mimeType [{}]: {}", mimeType, listOfCandidates);
    }
    Metacard generatedMetacard = null;
    // can create the metacard, then do not need to try any remaining InputTransformers.
    for (InputTransformer transformer : listOfCandidates) {
        try (InputStream inputStreamMessageCopy = tfbos.asByteSource().openStream()) {
            if (StringUtils.isEmpty(metacardId)) {
                generatedMetacard = transformer.transform(inputStreamMessageCopy);
            } else {
                generatedMetacard = transformer.transform(inputStreamMessageCopy, metacardId);
            }
        } catch (CatalogTransformerException e) {
            LOGGER.debug("Transformer [{}] could not create metacard.", transformer, e);
        } catch (IOException e) {
            LOGGER.debug("Could not open input stream", e);
        }
        if (generatedMetacard != null) {
            break;
        }
    }
    LOGGER.trace("EXITING: generateMetacard");
    return Optional.ofNullable(generatedMetacard);
}

Example 43 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class ImportCommand method executeWithSubject.

@Override
protected final Object executeWithSubject() throws Exception {
    int metacards = 0;
    int content = 0;
    int derivedContent = 0;
    File file = initImportFile(importFile);
    InputTransformer transformer = getServiceByFilter(InputTransformer.class, String.format("(%s=%s)", "id", DEFAULT_TRANSFORMER_ID)).orElseThrow(() -> new CatalogCommandRuntimeException("Could not get " + DEFAULT_TRANSFORMER_ID + " input transformer"));
    if (unsafe) {
        if (!force) {
            String input = session.readLine("This will import data with no check to see if data is modified/corrupt. Do you wish to continue? (y/N) ", null);
            if (!input.matches("^[yY][eE]?[sS]?$")) {
                console.println("ABORTED IMPORT.");
                return null;
            }
        }
        securityLogger.audit("Skipping validation check of imported data. There are no " + "guarantees of integrity or authenticity of the imported data." + "File being imported: {}", importFile);
    } else {
        if (StringUtils.isBlank(signatureFile)) {
            String message = "A signature file must be provided with import data";
            console.println(message);
            throw new CatalogCommandRuntimeException(message);
        }
        String alias = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty("org.codice.ddf.system.hostname"));
        try (FileInputStream fileIs = new FileInputStream(file);
            FileInputStream sigFileIs = new FileInputStream(signatureFile)) {
            if (verifier == null) {
                verifier = new DigitalSignature(security);
            }
            if (!verifier.verifyDigitalSignature(fileIs, sigFileIs, alias)) {
                throw new CatalogCommandRuntimeException("The provided data could not be verified");
            }
        }
    }
    securityLogger.audit("Called catalog:import command on the file: {}", importFile);
    console.println("Importing file");
    Instant start = Instant.now();
    try (InputStream fis = new FileInputStream(file);
        ZipInputStream zipInputStream = new ZipInputStream(fis)) {
        ZipEntry entry = zipInputStream.getNextEntry();
        while (entry != null) {
            String filename = entry.getName();
            if (filename.startsWith("META-INF")) {
                entry = zipInputStream.getNextEntry();
                continue;
            }
            String[] pathParts = filename.split("\\" + File.separator);
            if (pathParts.length < 5) {
                console.println("Entry is not valid! " + filename);
                entry = zipInputStream.getNextEntry();
                continue;
            }
            String id = pathParts[ID];
            String type = pathParts[TYPE];
            switch(type) {
                case "metacard":
                    {
                        String metacardName = pathParts[NAME];
                        Metacard metacard = null;
                        try {
                            metacard = transformer.transform(new UncloseableBufferedInputStreamWrapper(zipInputStream), id);
                        } catch (IOException | CatalogTransformerException e) {
                            LOGGER.debug("Could not transform metacard: {}", LogSanitizer.sanitize(id));
                            entry = zipInputStream.getNextEntry();
                            continue;
                        }
                        metacard = applyInjectors(metacard, attributeInjectors);
                        catalogProvider.create(new CreateRequestImpl(metacard));
                        metacards++;
                        break;
                    }
                case "content":
                    {
                        content++;
                        String contentFilename = pathParts[NAME];
                        ContentItem contentItem = new ContentItemImpl(id, new ZipEntryByteSource(new UncloseableBufferedInputStreamWrapper(zipInputStream)), null, contentFilename, entry.getSize(), null);
                        CreateStorageRequestImpl createStorageRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>());
                        storageProvider.create(createStorageRequest);
                        storageProvider.commit(createStorageRequest);
                        break;
                    }
                case "derived":
                    {
                        derivedContent++;
                        String qualifier = pathParts[NAME];
                        String derivedContentName = pathParts[DERIVED_NAME];
                        ContentItem contentItem = new ContentItemImpl(id, qualifier, new ZipEntryByteSource(new UncloseableBufferedInputStreamWrapper(zipInputStream)), null, derivedContentName, entry.getSize(), null);
                        CreateStorageRequestImpl createStorageRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>());
                        storageProvider.create(createStorageRequest);
                        storageProvider.commit(createStorageRequest);
                        break;
                    }
                default:
                    {
                        LOGGER.debug("Cannot interpret type of {}", LogSanitizer.sanitize(type));
                    }
            }
            entry = zipInputStream.getNextEntry();
        }
    } catch (Exception e) {
        printErrorMessage(String.format("Exception while importing metacards (%s)%nFor more information set the log level to INFO (log:set INFO org.codice.ddf.commands.catalog) ", e.getMessage()));
        LOGGER.info("Exception while importing metacards", e);
        throw e;
    }
    console.println("File imported successfully. Imported in: " + getFormattedDuration(start));
    console.println("Number of metacards imported: " + metacards);
    console.println("Number of content imported: " + content);
    console.println("Number of derived content imported: " + derivedContent);
    return null;
}

Example 44 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class CatalogComponentTest method testTransformMetacard.

@Test
public void testTransformMetacard() throws Exception {
    LOGGER.debug("Running testTransformMetacard()");
    MockEndpoint mock = getMockEndpoint("mock:result");
    mock.expectedMinimumMessageCount(1);
    // Mock a XML InputTransformer and register it in the OSGi Registry
    // (PojoSR)
    InputTransformer mockTransformer = getMockInputTransformer();
    Hashtable<String, String> props = new Hashtable<String, String>();
    props.put(MimeTypeToTransformerMapper.ID_KEY, "xml");
    props.put(MimeTypeToTransformerMapper.MIME_TYPE_KEY, "text/xml");
    bundleContext.registerService(InputTransformer.class.getName(), mockTransformer, props);
    // Mock the MimeTypeToTransformerMapper and register it in the OSGi
    // Registry (PojoSR)
    MimeTypeToTransformerMapper matchingService = mock(MimeTypeToTransformerMapper.class);
    // HUGH bundleContext.registerService(
    // MimeTypeToTransformerMapper.class.getName(), matchingService, null );
    catalogComponent.setMimeTypeToTransformerMapper(matchingService);
    // Mock the MimeTypeToTransformerMapper returning the mock XML
    // InputTransformer
    when(matchingService.findMatches(eq(InputTransformer.class), isA(MimeType.class))).thenReturn((List) Arrays.asList(mockTransformer));
    // Send in sample XML as InputStream to InputTransformer
    InputStream input = IOUtils.toInputStream(xmlInput);
    // Get the InputTransformer registered with the ID associated with the
    // <from> node in the Camel route
    InputTransformer transformer = getTransformer("text/xml", "identity");
    assertNotNull("InputTransformer for mimeType=text/xml&id=identity not found", transformer);
    // Transform the XML input into a Metacard
    Metacard metacard = transformer.transform(input);
    assertNotNull(metacard);
    assertMockEndpointsSatisfied();
}

Example 45 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class AbstractCatalogService method generateMetacard.

private Metacard generateMetacard(MimeType mimeType, String id, InputStream message, String transformerId) throws MetacardCreationException {
    Metacard generatedMetacard = null;
    List<InputTransformer> listOfCandidates = mimeTypeToTransformerMapper.findMatches(InputTransformer.class, mimeType);
    List<String> stackTraceList = new ArrayList<>();
    LOGGER.trace("Entering generateMetacard.");
    LOGGER.debug("List of matches for mimeType [{}]: {}", mimeType, listOfCandidates);
    try (TemporaryFileBackedOutputStream fileBackedOutputStream = new TemporaryFileBackedOutputStream()) {
        try {
            if (null != message) {
                IOUtils.copy(message, fileBackedOutputStream);
            } else {
                throw new MetacardCreationException("Could not copy bytes of content message.  Message was NULL.");
            }
        } catch (IOException e) {
            throw new MetacardCreationException("Could not copy bytes of content message.", e);
        }
        Iterator<InputTransformer> it = listOfCandidates.iterator();
        if (StringUtils.isNotEmpty(transformerId)) {
            BundleContext bundleContext = getBundleContext();
            Collection<ServiceReference<InputTransformer>> serviceReferences = bundleContext.getServiceReferences(InputTransformer.class, "(id=" + transformerId + ")");
            it = serviceReferences.stream().map(bundleContext::getService).iterator();
        }
        while (it.hasNext()) {
            InputTransformer transformer = it.next();
            try (InputStream inputStreamMessageCopy = fileBackedOutputStream.asByteSource().openStream()) {
                generatedMetacard = transformer.transform(inputStreamMessageCopy);
            } catch (CatalogTransformerException | IOException e) {
                List<String> stackTraces = Arrays.asList(ExceptionUtils.getRootCauseStackTrace(e));
                stackTraceList.add(String.format("Transformer [%s] could not create metacard.", transformer));
                stackTraceList.addAll(stackTraces);
                LOGGER.debug("Transformer [{}] could not create metacard.", transformer, e);
            }
            if (generatedMetacard != null) {
                break;
            }
        }
        if (generatedMetacard == null) {
            throw new MetacardCreationException(String.format("Could not create metacard with mimeType %s : %s", mimeType, StringUtils.join(stackTraceList, "\n")));
        }
        if (id != null) {
            generatedMetacard.setAttribute(new AttributeImpl(Metacard.ID, id));
        }
        LOGGER.debug("Metacard id is {}", generatedMetacard.getId());
    } catch (IOException e) {
        throw new MetacardCreationException("Could not create metacard.", e);
    } catch (InvalidSyntaxException e) {
        throw new MetacardCreationException("Could not determine transformer", e);
    }
    return generatedMetacard;
}