Examples with FeedDependencyDeltaResults com.thinkbiganalytics.metadata.api.op.FeedDependencyDeltaResults used on opensource projects

Example 1 with FeedDependencyDeltaResults

use of com.thinkbiganalytics.metadata.api.op.FeedDependencyDeltaResults in project kylo by Teradata.

the class TriggerFeed method createFlowFile.

private FlowFile createFlowFile(ProcessContext context, ProcessSession session, FeedPreconditionTriggerEvent event) {
    final String feedId = getFeedId(context);
    getLog().info("createFlowFile for Feed {}", new Object[] { feedId });
    FlowFile file = null;
    if (feedId != null) {
        FeedDependencyDeltaResults deltas = getProviderService(context).getProvider().getFeedDependentResultDeltas(feedId);
        if (deltas != null && deltas.getDependentFeedNames() != null && !deltas.getDependentFeedNames().isEmpty()) {
            file = session.create();
            try {
                List<String> keysToMatch = getMatchingExecutionContextKeys(context);
                getLog().info("Reducing the Execution Context to match {} keys ", new Object[] { StringUtils.join((keysToMatch)) });
                deltas.reduceExecutionContextToMatchingKeys(keysToMatch);
                String value = MAPPER.writeValueAsString(deltas);
                // add the json as an attr value?
                // file = session.putAttribute(file, ComponentAttributes.FEED_DEPENDENT_RESULT_DELTAS.key(), value);
                // write the json back to the flow file content
                file = session.write(file, new OutputStreamCallback() {

                    @Override
                    public void process(OutputStream outputStream) throws IOException {
                        outputStream.write(value.getBytes(StandardCharsets.UTF_8));
                    }
                });
            } catch (JsonProcessingException e) {
                getLog().warn("Failed to serialize feed dependency result deltas", e);
            // TODO Swallow the exception and produce the flow file anyway?
            }
        } else {
            getLog().debug("Found no dependent feeds");
        }
    }
    if (file == null) {
        file = session.get();
    }
    return file;
}

Example 2 with FeedDependencyDeltaResults

use of com.thinkbiganalytics.metadata.api.op.FeedDependencyDeltaResults in project kylo by Teradata.

the class FeedsController method getDependentResultDeltas.

@GET
@Path("{feedId}/depfeeds/delta")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the dependencies delta for the specified feed.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the dependencies deltas.", response = FeedDependencyDeltaResults.class), @ApiResponse(code = 500, message = "The feed could not be found.", response = RestResponseStatus.class) })
public FeedDependencyDeltaResults getDependentResultDeltas(@PathParam("feedId") final String feedIdStr) {
    LOG.info("Get feed dependencies  delta for {}", feedIdStr);
    com.thinkbiganalytics.metadata.api.feed.Feed.ID feedId = this.feedProvider.resolveFeed(feedIdStr);
    return this.metadata.read(() -> {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_FEEDS);
        FeedDependencyDeltaResults results = this.feedOpsProvider.getDependentDeltaResults(feedId, null);
        return results;
    });
}

Example 3 with FeedDependencyDeltaResults

use of com.thinkbiganalytics.metadata.api.op.FeedDependencyDeltaResults in project kylo by Teradata.

the class TriggerFeedTest method testFeedDependencyResultsExecutionContext.

@Test
public void testFeedDependencyResultsExecutionContext() {
    FeedDependencyDeltaResults deltaResults = new FeedDependencyDeltaResults();
    List<String> feedNames = new ArrayList<>();
    feedNames.add("category.feed_a");
    feedNames.add("category.feed_b");
    deltaResults.setDependentFeedNames(feedNames);
    Map<String, FeedDependencyDeltaResults.FeedJobExecutionData> jobData = new HashMap<>();
    feedNames.stream().forEach(feedName -> {
        Map<String, Object> executionContext = new HashMap<>();
        executionContext.put("param1", "test");
        executionContext.put("export.kylo.param2", "test2");
        deltaResults.addFeedExecutionContext(feedName, new Long(1), DateTime.now(), DateTime.now(), executionContext);
    });
    String executionContextKeys = "export.kylo, export.test, test2 ";
    List<String> list = new ArrayList<String>(Arrays.asList(executionContextKeys.trim().split("\\s*,\\s*")));
    deltaResults.reduceExecutionContextToMatchingKeys(list);
    // assert just the 1 property got sent to the execution context
    Assert.assertEquals(1, deltaResults.getLatestFeedJobExecutionContext().get(feedNames.get(0)).getExecutionContext().size());
    // validate JSON transform
    ObjectMapper MAPPER = new ObjectMapper();
    MAPPER.registerModule(new JodaModule());
    MAPPER.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true);
    MAPPER.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    try {
        String value = MAPPER.writeValueAsString(deltaResults);
        Assert.assertNotNull(value);
    } catch (JsonProcessingException e) {
        e.printStackTrace();
    }
}

Example 4 with FeedDependencyDeltaResults

use of com.thinkbiganalytics.metadata.api.op.FeedDependencyDeltaResults in project kylo by Teradata.

the class MetadataClientTest method testGetFeedDependencyDeltas.

// @Test
public void testGetFeedDependencyDeltas() {
    FeedDependencyDeltaResults props = client.getFeedDependencyDeltas("90056286-a3b0-493c-89a4-91cb1e7529b6");
    assertThat(props).isNotNull();
}

Example 5 with FeedDependencyDeltaResults

use of com.thinkbiganalytics.metadata.api.op.FeedDependencyDeltaResults in project kylo by Teradata.

the class JobRepoFeedOperationsProvider method getDependentDeltaResults.

@Override
public FeedDependencyDeltaResults getDependentDeltaResults(Feed.ID feedId, Set<String> props) {
    Feed feed = this.feedProvider.getFeed(feedId);
    if (feed != null) {
        String systemFeedName = FeedNameUtil.fullName(feed.getCategory().getSystemName(), feed.getName());
        FeedDependencyDeltaResults results = new FeedDependencyDeltaResults(feed.getId().toString(), systemFeedName);
        // find this feeds latest completion
        BatchJobExecution latest = jobExecutionProvider.findLatestCompletedJobForFeed(systemFeedName);
        // get the dependent feeds
        List<Feed> dependents = feed.getDependentFeeds();
        if (dependents != null) {
            for (Feed depFeed : dependents) {
                String depFeedSystemName = FeedNameUtil.fullName(depFeed.getCategory().getSystemName(), depFeed.getName());
                // find Completed feeds executed since time
                Set<BatchJobExecution> jobs = null;
                if (latest != null) {
                    jobs = (Set<BatchJobExecution>) jobExecutionProvider.findJobsForFeedCompletedSince(depFeedSystemName, latest.getStartTime());
                } else {
                    BatchJobExecution job = jobExecutionProvider.findLatestCompletedJobForFeed(depFeedSystemName);
                    if (job != null) {
                        jobs = new HashSet<>();
                        jobs.add(job);
                    }
                }
                if (jobs != null) {
                    for (BatchJobExecution job : jobs) {
                        DateTime endTime = job.getEndTime();
                        Map<String, String> executionContext = job.getJobExecutionContextAsMap();
                        Map<String, Object> map = new HashMap<>();
                        // filter the map
                        if (executionContext != null) {
                            // add those requested to the results map
                            for (Entry<String, String> entry : executionContext.entrySet()) {
                                if (props == null || props.isEmpty() || props.contains(entry.getKey())) {
                                    map.put(entry.getKey(), entry.getValue());
                                }
                            }
                        }
                        results.addFeedExecutionContext(depFeedSystemName, job.getJobExecutionId(), job.getStartTime(), endTime, map);
                    }
                } else {
                    results.getDependentFeedNames().add(depFeedSystemName);
                }
            }
        }
        return results;
    } else {
        throw new FeedNotFoundException(feedId);
    }
}