Search in sources :

Example 31 with FeedCategory

use of com.thinkbiganalytics.feedmgr.rest.model.FeedCategory in project kylo by Teradata.

the class NifiRestTest method testLoad.

// @Test
public void testLoad() {
    // setup constants for the test
    String templateName = "Data Ingest";
    int num = 10;
    String processGroupName = "LoadTest";
    String feedPrefix = "LT_";
    String inputType = "org.apache.nifi.processors.standard.GetFile";
    List<NifiProperty> templateProperties = new ArrayList<>();
    String schedulePeriod = "10 sec";
    String GET_FILE_PROCESSOR_NAME = "Poll filesystem";
    String UPDATE_PARAMETERS_PROCESSOR_NAME = "Update flow parameters";
    String INPUT_DIRECTORY_PROPERTY = "Input Directory";
    String SOURCE_PROPERTY = "source";
    String ENTITY_PROPERTY = "entity";
    try {
        TemplateDTO template = restClient.getTemplateByName(templateName);
        List<NifiProperty> propertyList = restClient.getPropertiesForTemplate(template.getId(), true);
        NifiProperty inputDirectory = NifiPropertyUtil.getProperty(GET_FILE_PROCESSOR_NAME, INPUT_DIRECTORY_PROPERTY, propertyList);
        NifiProperty entity = NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, SOURCE_PROPERTY, propertyList);
        NifiProperty source = NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, ENTITY_PROPERTY, propertyList);
        templateProperties.add(inputDirectory);
        templateProperties.add(entity);
        templateProperties.add(source);
        NifiProcessorSchedule schedule = new NifiProcessorSchedule();
        schedule.setSchedulingStrategy("TIMER_DRIVEN");
        schedule.setSchedulingPeriod(schedulePeriod);
        for (int i = 0; i < num; i++) {
            String feedName = feedPrefix + i;
            List<NifiProperty> instanceProperties = NifiPropertyUtil.copyProperties(templateProperties);
            // update the properties
            NifiPropertyUtil.getProperty(GET_FILE_PROCESSOR_NAME, INPUT_DIRECTORY_PROPERTY, instanceProperties).setValue("/tmp/" + feedName);
            NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, SOURCE_PROPERTY, instanceProperties).setValue(processGroupName);
            NifiPropertyUtil.getProperty(UPDATE_PARAMETERS_PROCESSOR_NAME, ENTITY_PROPERTY, instanceProperties).setValue(feedName);
            FeedMetadata feedMetadata = new FeedMetadata();
            feedMetadata.setCategory(new FeedCategory());
            feedMetadata.getCategory().setSystemName(processGroupName);
            feedMetadata.setSystemFeedName("feedPrefix + i");
            CreateFeedBuilder.newFeed(restClient, nifiFlowCache, feedMetadata, template.getId(), new PropertyExpressionResolver(), propertyDescriptorTransform, createFeedBuilderCache, templateConnectionUtil).inputProcessorType(inputType).feedSchedule(schedule).properties(instanceProperties).build();
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : FeedCategory(com.thinkbiganalytics.feedmgr.rest.model.FeedCategory) TemplateDTO(org.apache.nifi.web.api.dto.TemplateDTO) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) ArrayList(java.util.ArrayList) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) PropertyExpressionResolver(com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver) IOException(java.io.IOException) NifiProcessorSchedule(com.thinkbiganalytics.nifi.rest.model.NifiProcessorSchedule)

Example 32 with FeedCategory

use of com.thinkbiganalytics.feedmgr.rest.model.FeedCategory in project kylo by Teradata.

the class NifiRestTest method testCreateFeed.

// @Test
public void testCreateFeed() throws Exception {
    TemplateDTO templateDTO = restClient.getTemplateByName("New Data Ingest");
    String inputType = "org.apache.nifi.processors.standard.GetFile";
    NifiProcessorSchedule schedule = new NifiProcessorSchedule();
    schedule.setSchedulingStrategy("TIMER_DRIVEN");
    schedule.setSchedulingPeriod("10 sec");
    String inputPortName = "From Data Ingest Feed";
    String feedOutputPortName = "To Data Ingest";
    FeedMetadata feedMetadata = new FeedMetadata();
    feedMetadata.setCategory(new FeedCategory());
    feedMetadata.getCategory().setSystemName("online");
    feedMetadata.setSystemFeedName("Scotts Feed");
    CreateFeedBuilder.newFeed(restClient, nifiFlowCache, feedMetadata, templateDTO.getId(), new PropertyExpressionResolver(), propertyDescriptorTransform, createFeedBuilderCache, templateConnectionUtil).inputProcessorType(inputType).feedSchedule(schedule).addInputOutputPort(new InputOutputPort(inputPortName, feedOutputPortName)).build();
}
Also used : FeedCategory(com.thinkbiganalytics.feedmgr.rest.model.FeedCategory) TemplateDTO(org.apache.nifi.web.api.dto.TemplateDTO) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) InputOutputPort(com.thinkbiganalytics.nifi.feedmgr.InputOutputPort) PropertyExpressionResolver(com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver) NifiProcessorSchedule(com.thinkbiganalytics.nifi.rest.model.NifiProcessorSchedule)

Aggregations

FeedCategory (com.thinkbiganalytics.feedmgr.rest.model.FeedCategory)32 FeedMetadata (com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata)18 ImportTemplate (com.thinkbiganalytics.feedmgr.service.template.importing.model.ImportTemplate)8 Category (com.thinkbiganalytics.metadata.api.category.Category)8 ArrayList (java.util.ArrayList)7 FeedSummary (com.thinkbiganalytics.feedmgr.rest.model.FeedSummary)5 NifiProperty (com.thinkbiganalytics.nifi.rest.model.NifiProperty)5 Test (org.junit.Test)5 RegisteredTemplate (com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate)4 IOException (java.io.IOException)4 PropertyExpressionResolver (com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver)3 NifiFeed (com.thinkbiganalytics.feedmgr.rest.model.NifiFeed)3 UploadProgressMessage (com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage)3 HadoopSecurityGroup (com.thinkbiganalytics.metadata.api.security.HadoopSecurityGroup)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 Sets (com.google.common.collect.Sets)2 DefaultTableSchema (com.thinkbiganalytics.discovery.model.DefaultTableSchema)2 DefaultTag (com.thinkbiganalytics.discovery.model.DefaultTag)2 MetadataFieldAnnotationFieldNameResolver (com.thinkbiganalytics.feedmgr.MetadataFieldAnnotationFieldNameResolver)2 ImportComponent (com.thinkbiganalytics.feedmgr.rest.ImportComponent)2