Examples with MetadataProviderService com.thinkbiganalytics.nifi.core.api.metadata.MetadataProviderService used on opensource projects

Example 1 with MetadataProviderService

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProviderService in project kylo by Teradata.

the class GetFeedsHistoryReindex method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) {
    final ComponentLog logger = getLog();
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        flowFile = session.create();
    }
    logger.debug("Checking for feeds requiring reindexing historical data");
    try {
        MetadataProviderService metadataProviderService = getMetadataService(context);
        if ((metadataProviderService != null) && (metadataProviderService.getProvider() != null)) {
            String dateTimeOfCheck = String.valueOf(DateTime.now(DateTimeZone.UTC));
            FeedsForDataHistoryReindex feedsForHistoryReindexing = getMetadataService(context).getProvider().getFeedsForHistoryReindexing();
            if (feedsForHistoryReindexing != null) {
                logger.info("Found {} feeds requiring reindexing historical data", new Object[] { feedsForHistoryReindexing.getFeeds().size() });
                if (feedsForHistoryReindexing.getFeedCount() > 0) {
                    for (Feed feedForHistoryReindexing : feedsForHistoryReindexing.getFeeds()) {
                        Map<String, String> attributes = new HashMap<>();
                        attributes.put(FEED_ID_FOR_HISTORY_REINDEX_KEY, feedForHistoryReindexing.getId());
                        attributes.put(FEED_SYSTEM_NAME_FOR_HISTORY_REINDEX_KEY, feedForHistoryReindexing.getSystemName());
                        attributes.put(FEED_CATEGORY_SYSTEM_NAME_FOR_HISTORY_REINDEX_KEY, feedForHistoryReindexing.getCategory().getSystemName());
                        attributes.put(FEED_STATUS_FOR_HISTORY_REINDEX_KEY, feedForHistoryReindexing.getCurrentHistoryReindexingStatus().getHistoryReindexingState().toString());
                        attributes.put(FEED_LAST_MODIFIED_UTC_FOR_HISTORY_REINDEX_KEY, feedForHistoryReindexing.getCurrentHistoryReindexingStatus().getLastModifiedTimestamp().toString());
                        attributes.put(FEEDS_TOTAL_COUNT_FOR_HISTORY_REINDEX_KEY, String.valueOf(feedsForHistoryReindexing.getFeedCount()));
                        attributes.put(FEEDS_TOTAL_IDS_FOR_HISTORY_REINDEX_KEY, feedsForHistoryReindexing.getFeedIds().toString());
                        attributes.put(FEEDS_CHECK_TIME_UTC_FOR_HISTORY_REINDEX_KEY, dateTimeOfCheck);
                        // all attributes from parent flow file copied except uuid, creates a FORK event
                        FlowFile feedFlowFile = session.create(flowFile);
                        feedFlowFile = session.putAllAttributes(feedFlowFile, attributes);
                        session.transfer(feedFlowFile, REL_FOUND);
                        logger.info("Flow file created for reindexing feed's historical data: feed id {}, category name {}, feed name {}", new Object[] { FEED_ID_FOR_HISTORY_REINDEX_KEY, FEED_CATEGORY_SYSTEM_NAME_FOR_HISTORY_REINDEX_KEY, FEED_SYSTEM_NAME_FOR_HISTORY_REINDEX_KEY });
                    }
                    flowFile = session.putAttribute(flowFile, FEEDS_TOTAL_COUNT_FOR_HISTORY_REINDEX_KEY, String.valueOf(feedsForHistoryReindexing.getFeedCount()));
                    flowFile = session.putAttribute(flowFile, FEEDS_CHECK_TIME_UTC_FOR_HISTORY_REINDEX_KEY, dateTimeOfCheck);
                    // only for found case
                    session.transfer(flowFile, REL_ORIGINAL);
                } else {
                    // this will always be 0 here
                    flowFile = session.putAttribute(flowFile, FEEDS_TOTAL_COUNT_FOR_HISTORY_REINDEX_KEY, String.valueOf(feedsForHistoryReindexing.getFeedCount()));
                    // this will always be empty list here
                    flowFile = session.putAttribute(flowFile, FEEDS_TOTAL_IDS_FOR_HISTORY_REINDEX_KEY, feedsForHistoryReindexing.getFeedIds().toString());
                    flowFile = session.putAttribute(flowFile, FEEDS_CHECK_TIME_UTC_FOR_HISTORY_REINDEX_KEY, dateTimeOfCheck);
                    session.transfer(flowFile, REL_NOT_FOUND);
                }
            }
        } else {
            logger.error("Error checking for feeds requiring reindexing historical data. Check if Kylo services is running, and accessible from NiFi.");
            session.transfer(flowFile, REL_FAILURE);
        }
    } catch (Exception e) {
        logger.error("An exception was thrown during check for feeds requiring reindexing historical data: {}", new Object[] { e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 2 with MetadataProviderService

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProviderService in project kylo by Teradata.

the class GetTableDataTest method setUp.

/**
 * Initialize instance variables
 */
@Before
public void setUp() throws Exception {
    // Setup services
    final DBCPService jdbcService = new MockDBCPService();
    final MetadataProviderService metadataService = new MockMetadataService();
    // Setup test runner
    runner.addControllerService(JDBC_SERVICE_IDENTIFIER, jdbcService);
    runner.addControllerService(METADATA_SERVICE_IDENTIFIER, metadataService);
    runner.enableControllerService(jdbcService);
    runner.enableControllerService(metadataService);
    runner.setProperty(GetTableData.JDBC_SERVICE, JDBC_SERVICE_IDENTIFIER);
    runner.setProperty(CommonProperties.METADATA_SERVICE, METADATA_SERVICE_IDENTIFIER);
    runner.setProperty(GetTableData.TABLE_NAME, "mytable");
    runner.setProperty(GetTableData.TABLE_SPECS, "id\nfirst_name\nlast_name\nemail  \n last_updated  \n\n");
}

Example 3 with MetadataProviderService

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProviderService in project kylo by Teradata.

the class TriggerCleanupTest method setUp.

/**
 * Initialize instance variables.
 */
@Before
public void setUp() throws Exception {
    // Setup services
    final CleanupEventService cleanupService = new MockCleanupEventService();
    final MetadataProviderService metadataService = new MockMetadataProviderService();
    // Setup test runner
    runner.addControllerService(CLEANUP_SERVICE_IDENTIFIER, cleanupService);
    runner.addControllerService(METADATA_SERVICE_IDENTIFIER, metadataService);
    runner.enableControllerService(cleanupService);
    runner.enableControllerService(metadataService);
    runner.setProperty(TriggerCleanup.CLEANUP_SERVICE, CLEANUP_SERVICE_IDENTIFIER);
    runner.setProperty(TriggerCleanup.METADATA_SERVICE, METADATA_SERVICE_IDENTIFIER);
}

Example 4 with MetadataProviderService

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProviderService in project kylo by Teradata.

the class GetFeedsHistoryReindexTest method testGeneralExceptionGettingFeedsToReindex.

@Test
public void testGeneralExceptionGettingFeedsToReindex() throws Exception {
    final String METADATA_SERVICE_IDENTIFIER = "MockMetadataProviderService_GeneralExceptionGettingFeedsToReindex";
    final TestRunner runner = TestRunners.newTestRunner(GetFeedsHistoryReindex.class);
    final MetadataProviderService metadataService = new MockMetadataProviderService_GeneralExceptionGettingFeedsToReindex();
    runner.addControllerService(METADATA_SERVICE_IDENTIFIER, metadataService);
    runner.enableControllerService(metadataService);
    runner.setProperty(GetFeedsHistoryReindex.METADATA_SERVICE, METADATA_SERVICE_IDENTIFIER);
    runner.run(1);
    runner.assertQueueEmpty();
    runner.assertTransferCount(GetFeedsHistoryReindex.REL_FOUND, 0);
    runner.assertTransferCount(GetFeedsHistoryReindex.REL_NOT_FOUND, 0);
    runner.assertTransferCount(GetFeedsHistoryReindex.REL_FAILURE, 1);
    runner.assertTransferCount(GetFeedsHistoryReindex.REL_ORIGINAL, 0);
    List<MockFlowFile> results = runner.getFlowFilesForRelationship(GetFeedsHistoryReindex.REL_FAILURE);
    MockFlowFile resultFlowFile = results.get(0);
    resultFlowFile.assertAttributeNotExists(GetFeedsHistoryReindex.FEEDS_TOTAL_COUNT_FOR_HISTORY_REINDEX_KEY);
    resultFlowFile.assertAttributeNotExists(GetFeedsHistoryReindex.FEEDS_TOTAL_IDS_FOR_HISTORY_REINDEX_KEY);
    resultFlowFile.assertAttributeNotExists(GetFeedsHistoryReindex.FEEDS_CHECK_TIME_UTC_FOR_HISTORY_REINDEX_KEY);
}

Example 5 with MetadataProviderService

use of com.thinkbiganalytics.nifi.core.api.metadata.MetadataProviderService in project kylo by Teradata.

the class GetFeedsHistoryReindexTest method testErrorGettingFeedsToReindex.

@Test
public void testErrorGettingFeedsToReindex() throws Exception {
    final String METADATA_SERVICE_IDENTIFIER = "MockMetadataProviderService_ErrorGettingFeedsToReindex";
    final TestRunner runner = TestRunners.newTestRunner(GetFeedsHistoryReindex.class);
    final MetadataProviderService metadataService = new MockMetadataProviderService_ErrorGettingFeedsToReindex();
    runner.addControllerService(METADATA_SERVICE_IDENTIFIER, metadataService);
    runner.enableControllerService(metadataService);
    runner.setProperty(GetFeedsHistoryReindex.METADATA_SERVICE, METADATA_SERVICE_IDENTIFIER);
    runner.run(1);
    runner.assertQueueEmpty();
    runner.assertTransferCount(GetFeedsHistoryReindex.REL_FOUND, 0);
    runner.assertTransferCount(GetFeedsHistoryReindex.REL_NOT_FOUND, 0);
    runner.assertTransferCount(GetFeedsHistoryReindex.REL_FAILURE, 1);
    runner.assertTransferCount(GetFeedsHistoryReindex.REL_ORIGINAL, 0);
    List<MockFlowFile> results = runner.getFlowFilesForRelationship(GetFeedsHistoryReindex.REL_FAILURE);
    MockFlowFile resultFlowFile = results.get(0);
    resultFlowFile.assertAttributeNotExists(GetFeedsHistoryReindex.FEEDS_TOTAL_COUNT_FOR_HISTORY_REINDEX_KEY);
    resultFlowFile.assertAttributeNotExists(GetFeedsHistoryReindex.FEEDS_TOTAL_IDS_FOR_HISTORY_REINDEX_KEY);
    resultFlowFile.assertAttributeNotExists(GetFeedsHistoryReindex.FEEDS_CHECK_TIME_UTC_FOR_HISTORY_REINDEX_KEY);
}