Examples with InputTransformer ddf.catalog.transform.InputTransformer used on opensource projects

Example 51 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class OpenSearchSource method doQueryById.

private SourceResponse doQueryById(QueryRequest queryRequest, WebClient restWebClient) throws UnsupportedQueryException {
    InputStream responseStream = performRequest(restWebClient);
    try (TemporaryFileBackedOutputStream fileBackedOutputStream = new TemporaryFileBackedOutputStream()) {
        IOUtils.copyLarge(responseStream, fileBackedOutputStream);
        InputTransformer inputTransformer;
        try (InputStream inputStream = fileBackedOutputStream.asByteSource().openStream()) {
            inputTransformer = getInputTransformer(inputStream);
        }
        try (InputStream inputStream = fileBackedOutputStream.asByteSource().openStream()) {
            final Metacard metacard = inputTransformer.transform(inputStream);
            metacard.setSourceId(getId());
            ResultImpl result = new ResultImpl(metacard);
            List<Result> resultQueue = new ArrayList<>();
            resultQueue.add(result);
            return new SourceResponseImpl(queryRequest, resultQueue);
        }
    } catch (IOException | CatalogTransformerException e) {
        throw new UnsupportedQueryException("Problem with transformation.", e);
    }
}

Example 52 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class OpenSearchSource method lookupTransformerReference.

@Nullable
protected InputTransformer lookupTransformerReference(String namespaceUri) throws InvalidSyntaxException {
    LOGGER.trace("Looking up Input Transformer by schema : {}", namespaceUri);
    Bundle bundle = getBundle();
    if (bundle != null) {
        BundleContext bundleContext = bundle.getBundleContext();
        Collection<ServiceReference<InputTransformer>> transformerReferences = bundleContext.getServiceReferences(InputTransformer.class, "(schema=" + namespaceUri + ")");
        if (CollectionUtils.isNotEmpty(transformerReferences)) {
            ServiceReference<InputTransformer> transformer = transformerReferences.iterator().next();
            LOGGER.trace("Found Input Transformer {} by schema {}", transformer.getBundle().getSymbolicName(), namespaceUri);
            return bundleContext.getService(transformer);
        }
        LOGGER.trace("Failed to find Input Transformer by schema : {}", namespaceUri);
    }
    return null;
}

Example 53 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class TransformersCommandTest method testMultipleMimeTypes.

@Test
public void testMultipleMimeTypes() throws Exception {
    List<ServiceReference<InputTransformer>> serviceReferences = new ArrayList<>();
    ServiceReference<InputTransformer> serviceReference = mock(ServiceReference.class);
    List<String> mimeTypes = new ArrayList<>();
    mimeTypes.add("mime1");
    mimeTypes.add("mime2");
    mimeTypes.add("mime3");
    when(serviceReference.getProperty(SERVICE_ID)).thenReturn("Test");
    when(serviceReference.getProperty(MIME_TYPE)).thenReturn(mimeTypes);
    serviceReferences.add(serviceReference);
    when(bundleContext.getServiceReferences(InputTransformer.class, FILTER)).thenReturn(serviceReferences);
    transformersCommand.executeWithSubject();
    String output = consoleOutput.getOutput();
    assertThat(output, containsString("mime-types: mime1"));
    assertThat(output, containsString("mime2"));
    assertThat(output, containsString("mime3"));
}

Example 54 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class CatalogFrameworkImplTest method setup.

@Before
public void setup() throws StopProcessingException, PluginExecutionException, URISyntaxException, FederationException, IOException, CatalogTransformerException, InterruptedException {
    System.setProperty("bad.files", "crossdomain.xml,clientaccesspolicy.xml,.htaccess,.htpasswd,hosts,passwd,group,resolv.conf,nfs.conf,ftpd.conf,ntp.conf,web.config,robots.txt");
    System.setProperty("bad.file.extensions", ".exe,.jsp,.html,.js,.php,.phtml,.php3,.php4,.php5,.phps,.shtml,.jhtml,.pl,.py,.cgi,.msi,.com,.scr,.gadget,.application,.pif,.hta,.cpl,.msc,.jar,.kar,.bat,.cmd,.vb,.vbs,.vbe,.jse,.ws,.wsf,.wsc,.wsh,.ps1,.ps1xml,.ps2,.ps2xml,.psc1,.psc2,.msh,.msh1,.msh2,.mshxml,.msh1xml,.msh2xml,.scf,.lnk,.inf,.reg,.dll,.vxd,.cpl,.cfg,.config,.crt,.cert,.pem,.jks,.p12,.p7b,.key,.der,.csr,.jsb,.mhtml,.mht,.xhtml,.xht");
    System.setProperty("bad.mime.types", "text/html,text/javascript,text/x-javascript,application/x-shellscript,text/scriptlet,application/x-msdownload,application/x-msmetafile");
    System.setProperty("ignore.files", ".DS_Store,Thumbs.db");
    // Setup
    /*
     * Prepare to capture the ResourceResponse argument passed into
     * PostResourcePlugin.process(). We will verify that it contains a non-null ResourceRequest
     * in the verification section of this test.
     */
    argument = ArgumentCaptor.forClass(ResourceResponse.class);
    Resource mockResource = mock(Resource.class);
    mockResourceRequest = mock(ResourceRequest.class);
    when(mockResourceRequest.getAttributeValue()).thenReturn(new URI("myURI"));
    when(mockResourceRequest.getAttributeName()).thenReturn(new String("myName"));
    mockResourceResponse = mock(ResourceResponse.class);
    when(mockResourceResponse.getRequest()).thenReturn(mockResourceRequest);
    when(mockResourceResponse.getResource()).thenReturn(mockResource);
    mockPostResourcePlugin = mock(PostResourcePlugin.class);
    /*
     * We verify (see verification section of test) that PostResourcePlugin.process() receives a
     * ResourceResponse with a non-null ResourceRequest. We assume that it works correctly and
     * returns a ResourceResponse with a non-null ResourceRequest, so we return our
     * mockResouceResponse that contains a non-null ResourceRequest.
     */
    when(mockPostResourcePlugin.process(isA(ResourceResponse.class))).thenReturn(mockResourceResponse);
    List<PostResourcePlugin> mockPostResourcePlugins = new ArrayList<PostResourcePlugin>();
    mockPostResourcePlugins.add(mockPostResourcePlugin);
    eventAdmin = new MockEventProcessor();
    provider = new MockMemoryProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<>(), true, new Date());
    storageProvider = new MockMemoryStorageProvider();
    ArrayList<PostIngestPlugin> postIngestPlugins = new ArrayList<PostIngestPlugin>();
    postIngestPlugins.add(eventAdmin);
    mockFederationStrategy = mock(FederationStrategy.class);
    Result mockFederationResult = mock(Result.class);
    when(mockFederationResult.getMetacard()).thenReturn(new MetacardImpl());
    QueryRequest mockQueryRequest = mock(QueryRequest.class);
    Query mockQuery = mock(Query.class);
    when(mockQuery.getTimeoutMillis()).thenReturn(1L);
    when(mockQueryRequest.getQuery()).thenReturn(mockQuery);
    QueryResponseImpl queryResponse = new QueryResponseImpl(mockQueryRequest, Collections.singletonList(mockFederationResult), 1);
    when(mockFederationStrategy.federate(anyList(), any())).thenReturn(queryResponse);
    federatedSources = createDefaultFederatedSourceList(true);
    MimeTypeResolver mimeTypeResolver = mock(MimeTypeResolver.class);
    MimeTypeToTransformerMapper mimeTypeToTransformerMapper = mock(MimeTypeToTransformerMapper.class);
    InputTransformer inputTransformer = mock(InputTransformer.class);
    when(inputTransformer.transform(any(InputStream.class))).thenReturn(new MetacardImpl());
    when(mimeTypeToTransformerMapper.findMatches(any(Class.class), any(MimeType.class))).thenReturn(Collections.singletonList(inputTransformer));
    mockRemoteDeleteOperations = mock(RemoteDeleteOperations.class);
    FrameworkProperties frameworkProperties = new FrameworkProperties();
    frameworkProperties.setAccessPlugins(new ArrayList<>());
    frameworkProperties.setPolicyPlugins(new ArrayList<>());
    frameworkProperties.setCatalogProviders(Collections.singletonList((CatalogProvider) provider));
    frameworkProperties.setPostResource(mockPostResourcePlugins);
    frameworkProperties.setFederationStrategy(mockFederationStrategy);
    frameworkProperties.setFilterBuilder(new GeotoolsFilterBuilder());
    frameworkProperties.setPreIngest(new ArrayList<>());
    frameworkProperties.setPostIngest(postIngestPlugins);
    frameworkProperties.setPreQuery(new ArrayList<>());
    frameworkProperties.setPostQuery(new ArrayList<>());
    frameworkProperties.setPreResource(new ArrayList<>());
    frameworkProperties.setPostResource(new ArrayList<>());
    frameworkProperties.setQueryResponsePostProcessor(mock(QueryResponsePostProcessor.class));
    frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
    frameworkProperties.setMimeTypeMapper(new MimeTypeMapperImpl(Collections.singletonList(mimeTypeResolver)));
    frameworkProperties.setMimeTypeToTransformerMapper(mimeTypeToTransformerMapper);
    List<FederatedSource> federatedSourceList = new ArrayList<>();
    if (federatedSources != null) {
        for (FederatedSource source : federatedSources) {
            federatedSourceList.add(source);
        }
    }
    frameworkProperties.setFederatedSources(federatedSourceList);
    defaultAttributeValueRegistry = new DefaultAttributeValueRegistryImpl();
    frameworkProperties.setDefaultAttributeValueRegistry(defaultAttributeValueRegistry);
    attributeInjector = spy(new AttributeInjectorImpl(new AttributeRegistryImpl()));
    frameworkProperties.setAttributeInjectors(Collections.singletonList(attributeInjector));
    uuidGenerator = mock(UuidGenerator.class);
    when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
    sourceActionRegistry = mock(ActionRegistry.class);
    when(sourceActionRegistry.list(any())).thenReturn(Collections.emptyList());
    final SourcePoller<SourceStatus> mockStatusSourcePoller = mock(SourcePoller.class);
    doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).isAvailable() ? SourceStatus.AVAILABLE : SourceStatus.UNAVAILABLE)).when(mockStatusSourcePoller).getCachedValueForSource(any(Source.class));
    final SourcePoller<Set<ContentType>> mockContentTypesSourcePoller = mock(SourcePoller.class);
    doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).getContentTypes())).when(mockContentTypesSourcePoller).getCachedValueForSource(any(Source.class));
    OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
    MetacardFactory metacardFactory = new MetacardFactory(mimeTypeToTransformerMapper, uuidGenerator);
    OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
    SourceOperations sourceOperations = new SourceOperations(frameworkProperties, sourceActionRegistry, mockStatusSourcePoller, mockContentTypesSourcePoller);
    TransformOperations transformOperations = new TransformOperations(frameworkProperties);
    Historian historian = new Historian();
    historian.setHistoryEnabled(false);
    QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
    queryOperations.setSecurityLogger(mock(SecurityLogger.class));
    queryOperations.setPermissions(new PermissionsImpl());
    OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
    OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
    CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    UpdateOperations updateOperations = new UpdateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    deleteOperations = new DeleteOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard);
    ResourceOperations resOps = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity) {

        @Override
        protected ResourceInfo getResourceInfo(ResourceRequest resourceRequest, String site, boolean isEnterprise, StringBuilder federatedSite, Map<String, Serializable> requestProperties, boolean fanoutEnabled) throws ResourceNotSupportedException, ResourceNotFoundException {
            URI uri = null;
            Metacard metacard = new MetacardImpl();
            try {
                uri = new URI("myURI");
            } catch (URISyntaxException e) {
            }
            return new ResourceInfo(metacard, uri);
        }
    };
    updateOperations.setHistorian(historian);
    deleteOperations.setHistorian(historian);
    framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resOps, sourceOperations, transformOperations);
    // Conditionally bind objects if framework properties are setup
    if (!CollectionUtils.isEmpty(frameworkProperties.getCatalogProviders())) {
        sourceOperations.bind(provider);
    }
    sourceOperations.bind(storageProvider);
    resourceFramework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resOps, sourceOperations, transformOperations);
    // Conditionally bind objects if framework properties are setup
    if (!CollectionUtils.isEmpty(frameworkProperties.getCatalogProviders())) {
        sourceOperations.bind(provider);
    }
    sourceOperations.bind(storageProvider);
    ThreadContext.bind(mock(Subject.class));
}

Example 55 with InputTransformer

use of ddf.catalog.transform.InputTransformer in project ddf by codice.

the class FanoutCatalogFrameworkTest method testBlacklistedTagCreateStorageRequestFails.

@Test(expected = IngestException.class)
public void testBlacklistedTagCreateStorageRequestFails() throws Exception {
    Metacard metacard = new MetacardImpl();
    metacard.setAttribute(new AttributeImpl(Metacard.TAGS, "blacklisted"));
    CatalogProvider catalogProvider = mock(CatalogProvider.class);
    doReturn(true).when(catalogProvider).isAvailable();
    StorageProvider storageProvider = new MockMemoryStorageProvider();
    MimeTypeMapper mimeTypeMapper = mock(MimeTypeMapper.class);
    doReturn("extension").when(mimeTypeMapper).getFileExtensionForMimeType(anyString());
    InputTransformer transformer = mock(InputTransformer.class);
    doReturn(metacard).when(transformer).transform(any(InputStream.class));
    MimeTypeToTransformerMapper mimeTypeToTransformerMapper = mock(MimeTypeToTransformerMapper.class);
    doReturn(Collections.singletonList(transformer)).when(mimeTypeToTransformerMapper).findMatches(any(Class.class), any(MimeType.class));
    frameworkProperties.setCatalogProviders(Collections.singletonList(catalogProvider));
    frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
    frameworkProperties.setMimeTypeMapper(mimeTypeMapper);
    frameworkProperties.setMimeTypeToTransformerMapper(mimeTypeToTransformerMapper);
    OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
    MetacardFactory metacardFactory = new MetacardFactory(frameworkProperties.getMimeTypeToTransformerMapper(), uuidGenerator);
    OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
    final SourcePoller<SourceStatus> mockStatusSourcePoller = mock(SourcePoller.class);
    doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).isAvailable() ? SourceStatus.AVAILABLE : SourceStatus.UNAVAILABLE)).when(mockStatusSourcePoller).getCachedValueForSource(any(Source.class));
    SourceOperations sourceOperations = new SourceOperations(frameworkProperties, sourceActionRegistry, mockStatusSourcePoller, mock(SourcePoller.class));
    sourceOperations.bind(catalogProvider);
    sourceOperations.bind(storageProvider);
    TransformOperations transformOperations = new TransformOperations(frameworkProperties);
    QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
    OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
    OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
    ResourceOperations resourceOperations = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity);
    OperationsMetacardSupport opsMetacardSupport = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
    // Need to set these for InputValidation to work
    System.setProperty("bad.files", "none");
    System.setProperty("bad.file.extensions", "none");
    System.setProperty("bad.mime.types", "none");
    System.setProperty("ignore.files", "");
    CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacardSupport, opsCatStore, opsStorage);
    framework = new CatalogFrameworkImpl(createOperations, null, null, queryOperations, resourceOperations, sourceOperations, transformOperations);
    framework.setId(NEW_SOURCE_ID);
    framework.setFanoutEnabled(true);
    framework.setFanoutTagBlacklist(Collections.singletonList("blacklisted"));
    ContentItem item = new ContentItemImpl(uuidGenerator.generateUuid(), ByteSource.empty(), "text/xml", "filename.xml", 0L, metacard);
    CreateStorageRequest request = new CreateStorageRequestImpl(Collections.singletonList(item), new HashMap<>());
    framework.create(request);
}