use of com.evolveum.midpoint.prism.query.ObjectQuery in project midpoint by Evolveum.
the class AbstractSearchIterativeTaskHandler method runInternal.
public TaskRunResult runInternal(Task coordinatorTask) {
OperationResult opResult = new OperationResult(taskOperationPrefix + ".run");
opResult.setStatus(OperationResultStatus.IN_PROGRESS);
TaskRunResult runResult = new TaskRunResult();
runResult.setOperationResult(opResult);
H resultHandler;
try {
resultHandler = createHandler(runResult, coordinatorTask, opResult);
} catch (SecurityViolationException | SchemaException | RuntimeException e) {
LOGGER.error("{}: Error while creating a result handler: {}", taskName, e.getMessage(), e);
opResult.recordFatalError("Error while creating a result handler: " + e.getMessage(), e);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(coordinatorTask.getProgress());
return runResult;
}
if (resultHandler == null) {
// the error should already be in the runResult
return runResult;
}
// copying relevant configuration items from task to handler
resultHandler.setEnableIterationStatistics(isEnableIterationStatistics());
resultHandler.setEnableSynchronizationStatistics(isEnableSynchronizationStatistics());
resultHandler.setEnableActionsExecutedStatistics(isEnableActionsExecutedStatistics());
boolean cont = initializeRun(resultHandler, runResult, coordinatorTask, opResult);
if (!cont) {
return runResult;
}
// TODO: error checking - already running
if (coordinatorTask.getOid() == null) {
throw new IllegalArgumentException("Transient tasks cannot be run by " + AbstractSearchIterativeTaskHandler.class + ": " + coordinatorTask);
}
handlers.put(coordinatorTask.getOid(), resultHandler);
ObjectQuery query;
try {
query = createQuery(resultHandler, runResult, coordinatorTask, opResult);
} catch (SchemaException ex) {
logErrorAndSetResult(runResult, resultHandler, "Schema error while creating a search filter", ex, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
}
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("{}: using a query (before evaluating expressions):\n{}", taskName, DebugUtil.debugDump(query));
}
if (query == null) {
// the error should already be in the runResult
return runResult;
}
try {
// TODO consider which variables should go here (there's no focus, shadow, resource - only configuration)
if (ExpressionUtil.hasExpressions(query.getFilter())) {
PrismObject<SystemConfigurationType> configuration = systemObjectCache.getSystemConfiguration(opResult);
ExpressionVariables variables = Utils.getDefaultExpressionVariables(null, null, null, configuration != null ? configuration.asObjectable() : null);
try {
ExpressionEnvironment<?> env = new ExpressionEnvironment<>(coordinatorTask, opResult);
ModelExpressionThreadLocalHolder.pushExpressionEnvironment(env);
query = ExpressionUtil.evaluateQueryExpressions(query, variables, expressionFactory, prismContext, "evaluate query expressions", coordinatorTask, opResult);
} finally {
ModelExpressionThreadLocalHolder.popExpressionEnvironment();
}
}
} catch (SchemaException | ObjectNotFoundException | ExpressionEvaluationException e) {
logErrorAndSetResult(runResult, resultHandler, "Error while evaluating expressions in a search filter", e, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
}
Class<? extends ObjectType> type = getType(coordinatorTask);
Collection<SelectorOptions<GetOperationOptions>> queryOptions = createQueryOptions(resultHandler, runResult, coordinatorTask, opResult);
boolean useRepository = useRepositoryDirectly(resultHandler, runResult, coordinatorTask, opResult);
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("{}: searching {} with options {}, using query:\n{}", taskName, type, queryOptions, query.debugDump());
}
try {
// counting objects can be within try-catch block, because the handling is similar to handling errors within searchIterative
Long expectedTotal = null;
if (countObjectsOnStart) {
if (!useRepository) {
Integer expectedTotalInt = modelObjectResolver.countObjects(type, query, queryOptions, coordinatorTask, opResult);
if (expectedTotalInt != null) {
// conversion would fail on null
expectedTotal = (long) expectedTotalInt;
}
} else {
expectedTotal = (long) repositoryService.countObjects(type, query, opResult);
}
LOGGER.trace("{}: expecting {} objects to be processed", taskName, expectedTotal);
}
runResult.setProgress(0);
coordinatorTask.setProgress(0);
if (expectedTotal != null) {
coordinatorTask.setExpectedTotal(expectedTotal);
}
try {
coordinatorTask.savePendingModifications(opResult);
} catch (ObjectAlreadyExistsException e) {
// other exceptions are handled in the outer try block
throw new IllegalStateException("Unexpected ObjectAlreadyExistsException when updating task progress/expectedTotal", e);
}
resultHandler.createWorkerThreads(coordinatorTask, opResult);
if (!useRepository) {
modelObjectResolver.searchIterative((Class<O>) type, query, queryOptions, resultHandler, coordinatorTask, opResult);
} else {
// TODO think about this
repositoryService.searchObjectsIterative(type, query, (ResultHandler) resultHandler, null, false, opResult);
}
resultHandler.completeProcessing(coordinatorTask, opResult);
} catch (ObjectNotFoundException e) {
// This is bad. The resource does not exist. Permanent problem.
logErrorAndSetResult(runResult, resultHandler, "Object not found", e, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
} catch (CommunicationException e) {
// Error, but not critical. Just try later.
logErrorAndSetResult(runResult, resultHandler, "Communication error", e, OperationResultStatus.PARTIAL_ERROR, TaskRunResultStatus.TEMPORARY_ERROR);
return runResult;
} catch (SchemaException e) {
// Not sure about this. But most likely it is a misconfigured resource or connector
// It may be worth to retry. Error is fatal, but may not be permanent.
logErrorAndSetResult(runResult, resultHandler, "Error dealing with schema", e, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.TEMPORARY_ERROR);
return runResult;
} catch (RuntimeException e) {
// Can be anything ... but we can't recover from that.
// It is most likely a programming error. Does not make much sense to retry.
logErrorAndSetResult(runResult, resultHandler, "Internal error", e, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
} catch (ConfigurationException e) {
// Not sure about this. But most likely it is a misconfigured resource or connector
// It may be worth to retry. Error is fatal, but may not be permanent.
logErrorAndSetResult(runResult, resultHandler, "Configuration error", e, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.TEMPORARY_ERROR);
return runResult;
} catch (SecurityViolationException e) {
logErrorAndSetResult(runResult, resultHandler, "Security violation", e, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
} catch (ExpressionEvaluationException e) {
logErrorAndSetResult(runResult, resultHandler, "Expression error", e, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
}
// TODO: check last handler status
handlers.remove(coordinatorTask.getOid());
runResult.setProgress(resultHandler.getProgress());
runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
if (logFinishInfo) {
String finishMessage = "Finished " + taskName + " (" + coordinatorTask + "). ";
String statistics = "Processed " + resultHandler.getProgress() + " objects in " + resultHandler.getWallTime() / 1000 + " seconds, got " + resultHandler.getErrors() + " errors.";
if (resultHandler.getProgress() > 0) {
statistics += " Average time for one object: " + resultHandler.getAverageTime() + " milliseconds" + " (wall clock time average: " + resultHandler.getWallAverageTime() + " ms).";
}
if (!coordinatorTask.canRun()) {
statistics += " Task was interrupted during processing.";
}
opResult.createSubresult(taskOperationPrefix + ".statistics").recordStatus(OperationResultStatus.SUCCESS, statistics);
TaskHandlerUtil.appendLastFailuresInformation(taskOperationPrefix, coordinatorTask, opResult);
LOGGER.info("{}", finishMessage + statistics);
}
try {
finish(resultHandler, runResult, coordinatorTask, opResult);
} catch (SchemaException e) {
logErrorAndSetResult(runResult, resultHandler, "Schema error while finishing the run", e, OperationResultStatus.FATAL_ERROR, TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
}
LOGGER.trace("{} run finished (task {}, run result {})", taskName, coordinatorTask, runResult);
return runResult;
}
use of com.evolveum.midpoint.prism.query.ObjectQuery in project midpoint by Evolveum.
the class DeleteTaskHandler method runInternal.
public <O extends ObjectType> TaskRunResult runInternal(Task task) {
LOGGER.trace("Delete task run starting ({})", task);
long startTimestamp = System.currentTimeMillis();
OperationResult opResult = new OperationResult("DeleteTask.run");
opResult.setStatus(OperationResultStatus.IN_PROGRESS);
TaskRunResult runResult = new TaskRunResult();
runResult.setOperationResult(opResult);
opResult.setSummarizeErrors(true);
opResult.setSummarizePartialErrors(true);
opResult.setSummarizeSuccesses(true);
QueryType queryType;
PrismProperty<QueryType> objectQueryPrismProperty = task.getExtensionProperty(SchemaConstants.MODEL_EXTENSION_OBJECT_QUERY);
if (objectQueryPrismProperty != null && objectQueryPrismProperty.getRealValue() != null) {
queryType = objectQueryPrismProperty.getRealValue();
} else {
// For "foolproofness" reasons we really require a query. Even if it is "ALL" query.
LOGGER.error("No query parameter in {}", task);
opResult.recordFatalError("No query parameter in " + task);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
}
Class<O> objectType;
QName objectTypeName;
PrismProperty<QName> objectTypePrismProperty = task.getExtensionProperty(SchemaConstants.MODEL_EXTENSION_OBJECT_TYPE);
if (objectTypePrismProperty != null && objectTypePrismProperty.getRealValue() != null) {
objectTypeName = objectTypePrismProperty.getRealValue();
objectType = (Class<O>) ObjectTypes.getObjectTypeFromTypeQName(objectTypeName).getClassDefinition();
} else {
LOGGER.error("No object type parameter in {}", task);
opResult.recordFatalError("No object type parameter in " + task);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
}
ObjectQuery query;
try {
query = QueryJaxbConvertor.createObjectQuery(objectType, queryType, prismContext);
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Using object query from the task: {}", query.debugDump());
}
} catch (SchemaException ex) {
LOGGER.error("Schema error while creating a search filter: {}", new Object[] { ex.getMessage(), ex });
opResult.recordFatalError("Schema error while creating a search filter: " + ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
return runResult;
}
boolean optionRaw = true;
PrismProperty<Boolean> optionRawPrismProperty = task.getExtensionProperty(SchemaConstants.MODEL_EXTENSION_OPTION_RAW);
if (optionRawPrismProperty != null && optionRawPrismProperty.getRealValue() != null && !optionRawPrismProperty.getRealValue()) {
optionRaw = false;
}
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Deleting {}, raw={} using query:\n{}", new Object[] { objectType.getSimpleName(), optionRaw, query.debugDump() });
}
// TODO
boolean countObjectsOnStart = true;
long progress = 0;
Integer maxSize = 100;
ObjectPaging paging = ObjectPaging.createPaging(0, maxSize);
query.setPaging(paging);
query.setAllowPartialResults(true);
Collection<SelectorOptions<GetOperationOptions>> searchOptions = null;
ModelExecuteOptions execOptions = null;
if (optionRaw) {
searchOptions = SelectorOptions.createCollection(GetOperationOptions.createRaw());
execOptions = ModelExecuteOptions.createRaw();
}
try {
// counting objects can be within try-catch block, because the handling is similar to handling errors within searchIterative
Long expectedTotal = null;
if (countObjectsOnStart) {
Integer expectedTotalInt = modelService.countObjects(objectType, query, searchOptions, task, opResult);
LOGGER.trace("Expecting {} objects to be deleted", expectedTotal);
if (expectedTotalInt != null) {
// conversion would fail on null
expectedTotal = (long) expectedTotalInt;
}
}
runResult.setProgress(progress);
task.setProgress(progress);
if (expectedTotal != null) {
task.setExpectedTotal(expectedTotal);
}
try {
task.savePendingModifications(opResult);
} catch (ObjectAlreadyExistsException e) {
// other exceptions are handled in the outer try block
throw new IllegalStateException("Unexpected ObjectAlreadyExistsException when updating task progress/expectedTotal", e);
}
long progressLastUpdated = 0;
SearchResultList<PrismObject<O>> objects;
while (true) {
objects = modelService.searchObjects(objectType, query, searchOptions, task, opResult);
if (objects.isEmpty()) {
break;
}
int skipped = 0;
for (PrismObject<O> object : objects) {
if (!optionRaw && ShadowType.class.isAssignableFrom(objectType) && Boolean.TRUE == ((ShadowType) (object.asObjectable())).isProtectedObject()) {
LOGGER.debug("Skipping delete of protected object {}", object);
skipped++;
continue;
}
ObjectDelta<?> delta = ObjectDelta.createDeleteDelta(objectType, object.getOid(), prismContext);
String objectName = PolyString.getOrig(object.getName());
String objectDisplayName = StatisticsUtil.getDisplayName(object);
String objectOid = object.getOid();
task.recordIterativeOperationStart(objectName, objectDisplayName, objectTypeName, objectOid);
long objectDeletionStarted = System.currentTimeMillis();
try {
modelService.executeChanges(MiscSchemaUtil.createCollection(delta), execOptions, task, opResult);
task.recordIterativeOperationEnd(objectName, objectDisplayName, objectTypeName, objectOid, objectDeletionStarted, null);
} catch (Throwable t) {
task.recordIterativeOperationEnd(objectName, objectDisplayName, objectTypeName, objectOid, objectDeletionStarted, t);
// TODO we don't want to continue processing if an error occurs?
throw t;
}
progress++;
task.setProgressTransient(progress);
if (System.currentTimeMillis() - progressLastUpdated > PROGRESS_UPDATE_INTERVAL) {
task.setProgress(progress);
updateState(task);
progressLastUpdated = System.currentTimeMillis();
}
}
opResult.summarize();
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Search returned {} objects, {} skipped, progress: {}, result:\n{}", new Object[] { objects.size(), skipped, progress, opResult.debugDump() });
}
if (objects.size() == skipped) {
break;
}
}
} catch (ObjectAlreadyExistsException | ObjectNotFoundException | SchemaException | ExpressionEvaluationException | ConfigurationException | PolicyViolationException | SecurityViolationException e) {
LOGGER.error("{}", new Object[] { e.getMessage(), e });
opResult.recordFatalError("Object not found " + e.getMessage(), e);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
return runResult;
} catch (CommunicationException e) {
LOGGER.error("{}", new Object[] { e.getMessage(), e });
opResult.recordFatalError("Object not found " + e.getMessage(), e);
runResult.setRunResultStatus(TaskRunResultStatus.TEMPORARY_ERROR);
runResult.setProgress(progress);
return runResult;
}
runResult.setProgress(progress);
runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
opResult.summarize();
opResult.recordSuccess();
long wallTime = System.currentTimeMillis() - startTimestamp;
String finishMessage = "Finished delete (" + task + "). ";
String statistics = "Processed " + progress + " objects in " + wallTime / 1000 + " seconds.";
if (progress > 0) {
statistics += " Wall clock time average: " + ((float) wallTime / (float) progress) + " milliseconds";
}
opResult.createSubresult(DeleteTaskHandler.class.getName() + ".statistics").recordStatus(OperationResultStatus.SUCCESS, statistics);
LOGGER.info(finishMessage + statistics);
LOGGER.trace("Run finished (task {}, run result {})", new Object[] { task, runResult });
return runResult;
}
use of com.evolveum.midpoint.prism.query.ObjectQuery in project midpoint by Evolveum.
the class ReconciliationTaskHandler method scanForUnfinishedOperations.
/**
* Scans shadows for unfinished operations and tries to finish them.
* Returns false if the reconciliation was interrupted.
*/
private boolean scanForUnfinishedOperations(Task task, String resourceOid, ReconciliationTaskResult reconResult, OperationResult result) throws SchemaException, ObjectAlreadyExistsException, CommunicationException, ObjectNotFoundException, ConfigurationException, SecurityViolationException {
LOGGER.trace("Scan for unfinished operations starting");
OperationResult opResult = result.createSubresult(OperationConstants.RECONCILIATION + ".repoReconciliation");
opResult.addParam("reconciled", true);
ObjectQuery query = QueryBuilder.queryFor(ShadowType.class, prismContext).block().not().item(ShadowType.F_FAILED_OPERATION_TYPE).isNull().endBlock().and().item(ShadowType.F_RESOURCE_REF).ref(resourceOid).build();
List<PrismObject<ShadowType>> shadows = repositoryService.searchObjects(ShadowType.class, query, null, opResult);
// for this phase, obviously
task.setExpectedTotal((long) shadows.size());
LOGGER.trace("Found {} accounts that were not successfully processed.", shadows.size());
reconResult.setUnOpsCount(shadows.size());
long startedAll = System.currentTimeMillis();
int processedSuccess = 0, processedFailure = 0;
for (PrismObject<ShadowType> shadow : shadows) {
long started = System.currentTimeMillis();
task.recordIterativeOperationStart(shadow.asObjectable());
OperationResult provisioningResult = new OperationResult(OperationConstants.RECONCILIATION + ".finishOperation");
try {
RepositoryCache.enter();
ProvisioningOperationOptions options = ProvisioningOperationOptions.createCompletePostponed(false);
Utils.clearRequestee(task);
provisioningService.refreshShadow(shadow, options, task, provisioningResult);
// retryFailedOperation(shadow.asObjectable(), opResult);
task.recordIterativeOperationEnd(shadow.asObjectable(), started, null);
processedSuccess++;
} catch (Throwable ex) {
task.recordIterativeOperationEnd(shadow.asObjectable(), started, ex);
processedFailure++;
opResult.recordFatalError("Failed to finish operation with shadow: " + ObjectTypeUtil.toShortString(shadow.asObjectable()) + ". Reason: " + ex.getMessage(), ex);
Collection<? extends ItemDelta> modifications = PropertyDelta.createModificationReplacePropertyCollection(ShadowType.F_ATTEMPT_NUMBER, shadow.getDefinition(), shadow.asObjectable().getAttemptNumber() + 1);
try {
repositoryService.modifyObject(ShadowType.class, shadow.getOid(), modifications, provisioningResult);
task.recordObjectActionExecuted(shadow, null, null, ChangeType.MODIFY, SchemaConstants.CHANGE_CHANNEL_RECON_URI, null);
} catch (Exception e) {
task.recordObjectActionExecuted(shadow, null, null, ChangeType.MODIFY, SchemaConstants.CHANGE_CHANNEL_RECON_URI, e);
LoggingUtils.logException(LOGGER, "Failed to record finish operation failure with shadow: " + ObjectTypeUtil.toShortString(shadow.asObjectable()), e);
}
} finally {
task.markObjectActionExecutedBoundary();
RepositoryCache.exit();
}
// TODO record statistics as well
incrementAndRecordProgress(task, opResult);
if (!task.canRun()) {
break;
}
}
// for next phases, it looks strangely to see progress e.g. 2/1
task.setExpectedTotal(null);
// for each try the operation again
String message = "Processing unfinished operations done. Out of " + shadows.size() + " objects, " + processedSuccess + " were processed successfully and processing of " + processedFailure + " resulted in failure. " + "Total time spent: " + (System.currentTimeMillis() - startedAll) + " ms. " + (!task.canRun() ? "Was interrupted during processing." : "");
opResult.computeStatus();
result.createSubresult(opResult.getOperation() + ".statistics").recordStatus(opResult.getStatus(), message);
LOGGER.debug("{}. Result: {}", message, opResult.getStatus());
return task.canRun();
}
use of com.evolveum.midpoint.prism.query.ObjectQuery in project midpoint by Evolveum.
the class ReconciliationTaskHandler method performResourceReconciliation.
// returns false in case of execution interruption
private boolean performResourceReconciliation(PrismObject<ResourceType> resource, ObjectClassComplexTypeDefinition objectclassDef, ReconciliationTaskResult reconResult, Task coordinatorTask, OperationResult result) throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException {
boolean interrupted;
OperationResult opResult = result.createSubresult(OperationConstants.RECONCILIATION + ".resourceReconciliation");
// Instantiate result handler. This will be called with every search
// result in the following iterative search
SynchronizeAccountResultHandler handler = new SynchronizeAccountResultHandler(resource.asObjectable(), objectclassDef, "reconciliation", coordinatorTask, changeNotificationDispatcher, taskManager);
handler.setSourceChannel(SchemaConstants.CHANGE_CHANNEL_RECON);
handler.setStopOnError(false);
coordinatorTask.setExpectedTotal(null);
try {
ObjectQuery query = objectclassDef.createShadowSearchQuery(resource.getOid());
OperationResult searchResult = new OperationResult(OperationConstants.RECONCILIATION + ".searchIterative");
handler.createWorkerThreads(coordinatorTask, searchResult);
// note that progress is incremented within the handler, as it extends AbstractSearchIterativeResultHandler
provisioningService.searchObjectsIterative(ShadowType.class, query, null, handler, coordinatorTask, searchResult);
handler.completeProcessing(coordinatorTask, searchResult);
interrupted = !coordinatorTask.canRun();
opResult.computeStatus();
String message = "Processed " + handler.getProgress() + " account(s), got " + handler.getErrors() + " error(s)";
if (interrupted) {
message += "; was interrupted during processing";
}
if (handler.getProgress() > 0) {
message += ". Average time for one object: " + handler.getAverageTime() + " ms (wall clock time average: " + handler.getWallAverageTime() + " ms).";
}
OperationResultStatus resultStatus = OperationResultStatus.SUCCESS;
if (handler.getErrors() > 0) {
resultStatus = OperationResultStatus.PARTIAL_ERROR;
}
opResult.recordStatus(resultStatus, message);
LOGGER.info("Finished resource part of {} reconciliation: {}", resource, message);
reconResult.setResourceReconCount(handler.getProgress());
reconResult.setResourceReconErrors(handler.getErrors());
} catch (ConfigurationException | SecurityViolationException | SchemaException | CommunicationException | ObjectNotFoundException | ExpressionEvaluationException | RuntimeException | Error e) {
opResult.recordFatalError(e);
throw e;
}
return !interrupted;
}
use of com.evolveum.midpoint.prism.query.ObjectQuery in project midpoint by Evolveum.
the class ReindexTaskHandler method createQuery.
@Override
protected ObjectQuery createQuery(ReindexResultHandler handler, TaskRunResult runResult, Task task, OperationResult opResult) throws SchemaException {
ObjectQuery query = createQueryFromTask(handler, runResult, task, opResult);
LOGGER.info("Using query:\n{}", query.debugDump());
return query;
}
Aggregations