use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.
the class ControllerFacade method getProvenanceQuery.
/**
* Retrieves the results of a provenance query.
*
* @param provenanceId id
* @return the results of a provenance query
*/
public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) {
try {
// get the query to the provenance repository
final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser());
// ensure the query results could be found
if (querySubmission == null) {
throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged.");
}
// get the original query and the results
final Query query = querySubmission.getQuery();
final QueryResult queryResult = querySubmission.getResult();
// build the response
final ProvenanceDTO provenanceDto = new ProvenanceDTO();
final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO();
final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO();
// include the original request and results
provenanceDto.setRequest(requestDto);
provenanceDto.setResults(resultsDto);
// convert the original request
requestDto.setStartDate(query.getStartDate());
requestDto.setEndDate(query.getEndDate());
requestDto.setMinimumFileSize(query.getMinFileSize());
requestDto.setMaximumFileSize(query.getMaxFileSize());
requestDto.setMaxResults(query.getMaxResults());
if (query.getSearchTerms() != null) {
final Map<String, String> searchTerms = new HashMap<>();
for (final SearchTerm searchTerm : query.getSearchTerms()) {
searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue());
}
requestDto.setSearchTerms(searchTerms);
}
// convert the provenance
provenanceDto.setId(query.getIdentifier());
provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime());
provenanceDto.setExpiration(queryResult.getExpiration());
provenanceDto.setFinished(queryResult.isFinished());
provenanceDto.setPercentCompleted(queryResult.getPercentComplete());
// convert each event
final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults);
if (includeResults || queryResult.isFinished()) {
final List<ProvenanceEventDTO> events = new ArrayList<>();
for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) {
events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize)));
}
resultsDto.setProvenanceEvents(events);
}
if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) {
resultsDto.setTotalCount(requestDto.getMaxResults().longValue());
resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+");
} else {
resultsDto.setTotalCount(queryResult.getTotalHitCount());
resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
}
// include any errors
if (queryResult.getError() != null) {
final Set<String> errors = new HashSet<>();
errors.add(queryResult.getError());
resultsDto.setErrors(errors);
}
// set the generated timestamp
final Date now = new Date();
resultsDto.setGenerated(now);
resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime()));
// get the oldest available event time
final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1);
if (!firstEvent.isEmpty()) {
resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime()));
}
provenanceDto.setResults(resultsDto);
return provenanceDto;
} catch (final IOException ioe) {
throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe);
}
}
use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.
the class ControllerFacade method submitProvenance.
/**
* Submits a provenance query.
*
* @param provenanceDto dto
* @return provenance info
*/
public ProvenanceDTO submitProvenance(ProvenanceDTO provenanceDto) {
final ProvenanceRequestDTO requestDto = provenanceDto.getRequest();
// create the query
final Query query = new Query(provenanceDto.getId());
// if the request was specified
if (requestDto != null) {
// add each search term specified
final Map<String, String> searchTerms = requestDto.getSearchTerms();
if (searchTerms != null) {
for (final Map.Entry<String, String> searchTerm : searchTerms.entrySet()) {
SearchableField field;
field = SearchableFields.getSearchableField(searchTerm.getKey());
if (field == null) {
field = SearchableFields.newSearchableAttribute(searchTerm.getKey());
}
query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue()));
}
}
// specify the start date if specified
if (requestDto.getStartDate() != null) {
query.setStartDate(requestDto.getStartDate());
}
// ensure an end date is populated
if (requestDto.getEndDate() != null) {
query.setEndDate(requestDto.getEndDate());
}
// set the min/max file size
query.setMinFileSize(requestDto.getMinimumFileSize());
query.setMaxFileSize(requestDto.getMaximumFileSize());
// set the max results desired
query.setMaxResults(requestDto.getMaxResults());
}
// submit the query to the provenance repository
final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser());
// return the query with the results populated at this point
return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
}
use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.
the class TestPersistentProvenanceRepository method testNotAuthorizedQuery.
@Test(timeout = 10000)
public void testNotAuthorizedQuery() throws IOException, InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration config = createConfiguration();
config.setMaxRecordLife(5, TimeUnit.MINUTES);
config.setMaxStorageCapacity(1024L * 1024L);
config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
// force new index to be created for each rollover
config.setDesiredIndexSize(10);
repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {
@Override
public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) {
return event.getEventId() > 2;
}
};
repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
final String uuid = "00000000-0000-0000-0000-000000000000";
final Map<String, String> attributes = new HashMap<>();
attributes.put("abc", "xyz");
attributes.put("xyz", "abc");
attributes.put("filename", "file-" + uuid);
final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
builder.setEventTime(System.currentTimeMillis());
builder.setEventType(ProvenanceEventType.RECEIVE);
builder.setTransitUri("nifi://unit-test");
builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
for (int i = 0; i < 10; i++) {
attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
// make sure the events are destroyed when we call purge
builder.setEventTime(10L);
repo.registerEvent(builder.build());
}
repo.waitForRollover();
final Query query = new Query("1234");
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234"));
final QuerySubmission submission = repo.submitQuery(query, createUser());
final QueryResult result = submission.getResult();
while (!result.isFinished()) {
Thread.sleep(100L);
}
// Ensure that we gets events with ID's 3 through 10.
final List<ProvenanceEventRecord> events = result.getMatchingEvents();
assertEquals(7, events.size());
final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList());
for (int i = 0; i < 7; i++) {
Assert.assertEquals(i + 3, eventIds.get(i).intValue());
}
}
use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.
the class TestLuceneEventIndex method testUnauthorizedEventsGetFilteredForQuery.
@Test(timeout = 60000)
public void testUnauthorizedEventsGetFilteredForQuery() throws InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration repoConfig = createConfig(1);
repoConfig.setDesiredIndexSize(1L);
final IndexManager indexManager = new SimpleIndexManager(repoConfig);
final ArrayListEventStore eventStore = new ArrayListEventStore();
final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
index.initialize(eventStore);
for (int i = 0; i < 3; i++) {
final ProvenanceEventRecord event = createEvent("1234");
final StorageResult storageResult = eventStore.addEvent(event);
index.addEvents(storageResult.getStorageLocations());
}
final Query query = new Query(UUID.randomUUID().toString());
final EventAuthorizer authorizer = new EventAuthorizer() {
@Override
public boolean isAuthorized(ProvenanceEventRecord event) {
return event.getEventId() % 2 == 0;
}
@Override
public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
throw new AccessDeniedException();
}
};
List<ProvenanceEventRecord> events = Collections.emptyList();
while (events.size() < 2) {
final QuerySubmission submission = index.submitQuery(query, authorizer, "unit test");
assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
events = submission.getResult().getMatchingEvents();
Thread.sleep(25L);
}
assertEquals(2, events.size());
}
use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.
the class PersistentProvenanceRepository method queryEvents.
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
final QuerySubmission submission = submitQuery(query, user);
final QueryResult result = submission.getResult();
while (!result.isFinished()) {
try {
Thread.sleep(100L);
} catch (final InterruptedException ie) {
Thread.currentThread().interrupt();
}
}
if (result.getError() != null) {
throw new IOException(result.getError());
}
logger.info("{} got {} hits", query, result.getTotalHitCount());
return result;
}
Aggregations