Examples with InputTransformer ddf.catalog.transform.InputTransformer used on opensource projects

Example 16 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class TestRestEndpoint method testParseAttachments.

@Test
public void testParseAttachments() throws IOException, CatalogTransformerException, SourceUnavailableException, IngestException, InvalidSyntaxException, MimeTypeResolutionException, URISyntaxException {
    CatalogFramework framework = givenCatalogFramework(SAMPLE_ID);
    BundleContext bundleContext = mock(BundleContext.class);
    Collection<ServiceReference<InputTransformer>> serviceReferences = new ArrayList<>();
    ServiceReference serviceReference = mock(ServiceReference.class);
    InputTransformer inputTransformer = mock(InputTransformer.class);
    MetacardImpl metacard = new MetacardImpl();
    metacard.setMetadata("Some Text Again");
    when(inputTransformer.transform(any())).thenReturn(metacard);
    when(bundleContext.getService(serviceReference)).thenReturn(inputTransformer);
    serviceReferences.add(serviceReference);
    when(bundleContext.getServiceReferences(InputTransformer.class, "(id=xml)")).thenReturn(serviceReferences);
    RESTEndpoint rest = new RESTEndpoint(framework) {

        @Override
        BundleContext getBundleContext() {
            return bundleContext;
        }
    };
    rest.setMetacardTypes(Collections.singletonList(BasicTypes.BASIC_METACARD));
    MimeTypeMapper mimeTypeMapper = mock(MimeTypeMapper.class);
    when(mimeTypeMapper.getMimeTypeForFileExtension("txt")).thenReturn("text/plain");
    when(mimeTypeMapper.getMimeTypeForFileExtension("xml")).thenReturn("text/xml");
    rest.setMimeTypeMapper(mimeTypeMapper);
    addMatchingService(rest, Arrays.asList(getSimpleTransformer()));
    List<Attachment> attachments = new ArrayList<>();
    ContentDisposition contentDisposition = new ContentDisposition("form-data; name=parse.resource; filename=C:\\DDF\\metacard.txt");
    Attachment attachment = new Attachment("parse.resource", new ByteArrayInputStream("Some Text".getBytes()), contentDisposition);
    attachments.add(attachment);
    ContentDisposition contentDisposition1 = new ContentDisposition("form-data; name=parse.metadata; filename=C:\\DDF\\metacard.xml");
    Attachment attachment1 = new Attachment("parse.metadata", new ByteArrayInputStream("Some Text Again".getBytes()), contentDisposition1);
    attachments.add(attachment1);
    RESTEndpoint.CreateInfo createInfo = rest.parseAttachments(attachments, "xml");
    assertThat(createInfo.getMetacard().getMetadata(), equalTo("Some Text Again"));
    ContentDisposition contentDisposition2 = new ContentDisposition("form-data; name=metadata; filename=C:\\DDF\\metacard.xml");
    Attachment attachment2 = new Attachment("metadata", new ByteArrayInputStream("<meta>beta</meta>".getBytes()), contentDisposition2);
    attachments.add(attachment2);
    ContentDisposition contentDisposition3 = new ContentDisposition("form-data; name=foo; filename=C:\\DDF\\metacard.xml");
    Attachment attachment3 = new Attachment("foo", new ByteArrayInputStream("bar".getBytes()), contentDisposition3);
    attachments.add(attachment3);
    createInfo = rest.parseAttachments(attachments, "xml");
    assertThat(createInfo.getMetacard().getMetadata(), equalTo("<meta>beta</meta>"));
    assertThat(createInfo.getMetacard().getAttribute("foo"), equalTo(null));
}

Example 17 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class TestRestEndpoint method testAddDocumentWithMetadataPositiveCase.

@Test
public void testAddDocumentWithMetadataPositiveCase() throws IOException, CatalogTransformerException, IngestException, SourceUnavailableException, URISyntaxException, InvalidSyntaxException, MimeTypeResolutionException {
    CatalogFramework framework = givenCatalogFramework(SAMPLE_ID);
    HttpHeaders headers = createHeaders(Arrays.asList(MediaType.APPLICATION_JSON));
    BundleContext bundleContext = mock(BundleContext.class);
    Collection<ServiceReference<InputTransformer>> serviceReferences = new ArrayList<>();
    ServiceReference serviceReference = mock(ServiceReference.class);
    InputTransformer inputTransformer = mock(InputTransformer.class);
    when(inputTransformer.transform(any())).thenReturn(new MetacardImpl());
    when(bundleContext.getService(serviceReference)).thenReturn(inputTransformer);
    serviceReferences.add(serviceReference);
    when(bundleContext.getServiceReferences(InputTransformer.class, "(id=xml)")).thenReturn(serviceReferences);
    RESTEndpoint rest = new RESTEndpoint(framework) {

        @Override
        BundleContext getBundleContext() {
            return bundleContext;
        }
    };
    UuidGenerator uuidGenerator = mock(UuidGenerator.class);
    when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
    rest.setUuidGenerator(uuidGenerator);
    rest.setMetacardTypes(Collections.singletonList(BasicTypes.BASIC_METACARD));
    MimeTypeMapper mimeTypeMapper = mock(MimeTypeMapper.class);
    when(mimeTypeMapper.getMimeTypeForFileExtension("txt")).thenReturn("text/plain");
    when(mimeTypeMapper.getMimeTypeForFileExtension("xml")).thenReturn("text/xml");
    rest.setMimeTypeMapper(mimeTypeMapper);
    addMatchingService(rest, Arrays.asList(getSimpleTransformer()));
    UriInfo info = givenUriInfo(SAMPLE_ID);
    List<Attachment> attachments = new ArrayList<>();
    ContentDisposition contentDisposition = new ContentDisposition("form-data; name=parse.resource; filename=C:\\DDF\\metacard.txt");
    Attachment attachment = new Attachment("parse.resource", new ByteArrayInputStream("Some Text".getBytes()), contentDisposition);
    attachments.add(attachment);
    ContentDisposition contentDisposition1 = new ContentDisposition("form-data; name=parse.metadata; filename=C:\\DDF\\metacard.xml");
    Attachment attachment1 = new Attachment("parse.metadata", new ByteArrayInputStream("Some Text Again".getBytes()), contentDisposition1);
    attachments.add(attachment1);
    ContentDisposition contentDisposition2 = new ContentDisposition("form-data; name=metadata; filename=C:\\DDF\\metacard.xml");
    Attachment attachment2 = new Attachment("metadata", new ByteArrayInputStream("<meta>beta</meta>".getBytes()), contentDisposition2);
    attachments.add(attachment2);
    MultipartBody multipartBody = new MultipartBody(attachments);
    Response response = rest.addDocument(headers, info, mock(HttpServletRequest.class), multipartBody, null, new ByteArrayInputStream("".getBytes()));
    LOGGER.debug(ToStringBuilder.reflectionToString(response));
    assertThat(response.getStatus(), equalTo(201));
    assertThat(response.getMetadata(), notNullValue());
    assertThat(response.getMetadata().get(Metacard.ID).get(0).toString(), equalTo(SAMPLE_ID));
}

Example 18 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class TestOpenSearchSource method getMockInputTransformer.

protected InputTransformer getMockInputTransformer() {
    InputTransformer inputTransformer = mock(InputTransformer.class);
    Metacard generatedMetacard = getSimpleMetacard();
    try {
        when(inputTransformer.transform(isA(InputStream.class))).thenReturn(generatedMetacard);
        when(inputTransformer.transform(isA(InputStream.class), isA(String.class))).thenReturn(generatedMetacard);
    } catch (IOException e) {
        fail();
    } catch (CatalogTransformerException e) {
        fail();
    }
    return inputTransformer;
}

Example 19 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class ImportCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    int metacards = 0;
    int content = 0;
    int derivedContent = 0;
    ZipValidator zipValidator = initZipValidator();
    File file = initImportFile(importFile);
    InputTransformer transformer = getServiceByFilter(InputTransformer.class, String.format("(%s=%s)", "id", DEFAULT_TRANSFORMER_ID)).orElseThrow(() -> new CatalogCommandRuntimeException("Could not get " + DEFAULT_TRANSFORMER_ID + " input transformer"));
    if (unsafe) {
        if (!force) {
            console.println("This will import data with no check to see if data is modified/corrupt. Do you wish to continue?");
            String input = getUserInputModifiable().toString();
            if (!input.matches("^[yY][eE]?[sS]?$")) {
                console.println("ABORTED IMPORT.");
                return null;
            }
        }
        SecurityLogger.audit("Skipping validation check of imported data. There are no " + "guarantees of integrity or authenticity of the imported data." + "File being imported: {}", importFile);
    } else {
        if (!zipValidator.validateZipFile(importFile)) {
            throw new CatalogCommandRuntimeException("Signature on zip file is not valid");
        }
    }
    SecurityLogger.audit("Called catalog:import command on the file: {}", importFile);
    console.println("Importing file");
    Instant start = Instant.now();
    try (InputStream fis = new FileInputStream(file);
        ZipInputStream zipInputStream = new ZipInputStream(fis)) {
        ZipEntry entry = zipInputStream.getNextEntry();
        while (entry != null) {
            String filename = entry.getName();
            if (filename.startsWith("META-INF")) {
                entry = zipInputStream.getNextEntry();
                continue;
            }
            String[] pathParts = filename.split("\\" + File.separator);
            if (pathParts.length < 5) {
                console.println("Entry is not valid! " + filename);
                entry = zipInputStream.getNextEntry();
                continue;
            }
            String id = pathParts[ID];
            String type = pathParts[TYPE];
            switch(type) {
                case "metacard":
                    {
                        metacards++;
                        String metacardName = pathParts[NAME];
                        Metacard metacard = null;
                        try {
                            metacard = transformer.transform(new UncloseableBufferedInputStreamWrapper(zipInputStream), id);
                        } catch (IOException | CatalogTransformerException e) {
                            LOGGER.debug("Could not transform metacard: {}", id);
                        }
                        catalogProvider.create(new CreateRequestImpl(metacard));
                        break;
                    }
                case "content":
                    {
                        content++;
                        String contentFilename = pathParts[NAME];
                        ContentItem contentItem = new ContentItemImpl(id, new ZipEntryByteSource(new UncloseableBufferedInputStreamWrapper(zipInputStream)), null, contentFilename, entry.getSize(), null);
                        CreateStorageRequestImpl createStorageRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>());
                        storageProvider.create(createStorageRequest);
                        storageProvider.commit(createStorageRequest);
                        break;
                    }
                case "derived":
                    {
                        derivedContent++;
                        String qualifier = pathParts[NAME];
                        String derivedContentName = pathParts[DERIVED_NAME];
                        ContentItem contentItem = new ContentItemImpl(id, qualifier, new ZipEntryByteSource(new UncloseableBufferedInputStreamWrapper(zipInputStream)), null, derivedContentName, entry.getSize(), null);
                        CreateStorageRequestImpl createStorageRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>());
                        storageProvider.create(createStorageRequest);
                        storageProvider.commit(createStorageRequest);
                        break;
                    }
                default:
                    {
                        LOGGER.debug("Cannot interpret type of {}", type);
                    }
            }
            entry = zipInputStream.getNextEntry();
        }
    } catch (Exception e) {
        printErrorMessage(String.format("Exception while importing metacards (%s)%nFor more information set the log level to INFO (log:set INFO org.codice.ddf.commands.catalog) ", e.getMessage()));
        LOGGER.info("Exception while importing metacards", e);
        throw e;
    }
    console.println("File imported successfully. Imported in: " + getFormattedDuration(start));
    console.println("Number of metacards imported: " + metacards);
    console.println("Number of content imported: " + content);
    console.println("Number of derived content imported: " + derivedContent);
    return null;
}

Example 20 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class OpenSearchSource method getInputTransformer.

private InputTransformer getInputTransformer(InputStream inputStream) {
    XMLStreamReader xmlStreamReader = null;
    try {
        xmlStreamReader = xmlInputFactory.createXMLStreamReader(inputStream);
        while (xmlStreamReader.hasNext()) {
            int next = xmlStreamReader.next();
            if (next == XMLStreamConstants.START_ELEMENT) {
                String namespaceUri = xmlStreamReader.getNamespaceURI();
                InputTransformer transformerReference = lookupTransformerReference(namespaceUri);
                if (transformerReference != null) {
                    return transformerReference;
                }
            }
        }
    } catch (XMLStreamException | InvalidSyntaxException e) {
        LOGGER.debug("Failed to parse transformer namespace", e);
    } finally {
        try {
            if (xmlStreamReader != null) {
                xmlStreamReader.close();
            }
        } catch (XMLStreamException e) {
            LOGGER.debug("failed to close namespace reader", e);
        }
    }
    return null;
}