Examples with WorkBucketType com.evolveum.midpoint.xml.ns._public.common.common_3.WorkBucketType used on opensource projects

Example 1 with WorkBucketType

use of com.evolveum.midpoint.xml.ns._public.common.common_3.WorkBucketType in project midpoint by Evolveum.

the class RepoConcurrencyTest method test150WorkBucketsReplace.

/**
 * Here we test concurrent work bucket delegation using
 * {@link RepositoryService#modifyObjectDynamically(Class, String, Collection, ModificationsSupplier, RepoModifyOptions, OperationResult)}
 * method.
 */
@Test
public void test150WorkBucketsReplace() throws Exception {
    int THREADS = 8;
    long DURATION = 30_000L;
    TaskType task = new TaskType(prismContext).name("test150").beginActivityState().beginActivity().beginBucketing().bucketsProcessingRole(BucketsProcessingRoleType.COORDINATOR).<ActivityStateType>end().<TaskActivityStateType>end().end();
    OperationResult result = new OperationResult("test150WorkBucketsReplace");
    String oid = plainRepositoryService.addObject(task.asPrismObject(), null, result);
    displayValue("object added", oid);
    logger.info("Starting worker threads");
    List<WorkerThread> threads = new ArrayList<>();
    for (int i = 0; i < THREADS; i++) {
        final int threadIndex = i;
        WorkerThread thread = new WorkerThread(i) {

            @Override
            void runOnce(OperationResult result) throws Exception {
                ModificationsSupplier<TaskType> modificationSupplier = task -> prismContext.deltaFor(TaskType.class).item(TaskType.F_ACTIVITY_STATE, F_ACTIVITY, F_BUCKETING, F_BUCKET).replaceRealValues(getBucketsToReplace(task)).asItemDeltas();
                plainRepositoryService.modifyObjectDynamically(TaskType.class, oid, null, modificationSupplier, null, result);
            }

            private List<WorkBucketType> getBucketsToReplace(TaskType task) {
                List<WorkBucketType> currentBuckets = getCurrentBuckets(task);
                List<WorkBucketType> newBuckets = CloneUtil.cloneCollectionMembers(currentBuckets);
                newBuckets.add(getNextBucket(currentBuckets));
                return newBuckets;
            }

            private WorkBucketType getNextBucket(List<WorkBucketType> currentBuckets) {
                return new WorkBucketType(prismContext).sequentialNumber(getLastBucketNumber(currentBuckets) + 1).state(WorkBucketStateType.DELEGATED).workerRef(String.valueOf(threadIndex), TaskType.COMPLEX_TYPE);
            }

            private List<WorkBucketType> getCurrentBuckets(TaskType task) {
                return task.getActivityState().getActivity().getBucketing().getBucket();
            }

            private int getLastBucketNumber(List<WorkBucketType> buckets) {
                return buckets.stream().mapToInt(WorkBucketType::getSequentialNumber).max().orElse(0);
            }

            @Override
            String description() {
                return "Bucket computer thread #" + threadIndex;
            }
        };
        thread.start();
        threads.add(thread);
    }
    waitForThreads(threads, DURATION);
    PrismObject<TaskType> taskAfter = plainRepositoryService.getObject(TaskType.class, oid, null, result);
    displayValue("task after", taskAfter);
    assertCorrectBucketSequence(taskAfter.asObjectable().getActivityState().getActivity().getBucketing().getBucket());
}

Example 2 with WorkBucketType

use of com.evolveum.midpoint.xml.ns._public.common.common_3.WorkBucketType in project midpoint by Evolveum.

the class RepoConcurrencyTest method test140WorkBucketsAdd.

/**
 * Here we test concurrent work bucket creation using
 * {@link RepositoryService#modifyObjectDynamically(Class, String, Collection, ModificationsSupplier, RepoModifyOptions, OperationResult)}
 * method.
 */
@Test
public void test140WorkBucketsAdd() throws Exception {
    int THREADS = 8;
    long DURATION = 30_000L;
    TaskType task = new TaskType(prismContext).name("test140").beginActivityState().beginActivity().beginBucketing().bucketsProcessingRole(BucketsProcessingRoleType.COORDINATOR).<ActivityStateType>end().<TaskActivityStateType>end().end();
    OperationResult result = new OperationResult("test140WorkBucketsAdd");
    String oid = plainRepositoryService.addObject(task.asPrismObject(), null, result);
    displayValue("object added", oid);
    logger.info("Starting worker threads");
    List<WorkerThread> threads = new ArrayList<>();
    for (int i = 0; i < THREADS; i++) {
        final int threadIndex = i;
        WorkerThread thread = new WorkerThread(i) {

            @Override
            void runOnce(OperationResult result) throws Exception {
                ModificationsSupplier<TaskType> modificationSupplier = task -> prismContext.deltaFor(TaskType.class).item(TaskType.F_ACTIVITY_STATE, F_ACTIVITY, F_BUCKETING, F_BUCKET).add(getNextBucket(task)).asItemDeltas();
                plainRepositoryService.modifyObjectDynamically(TaskType.class, oid, null, modificationSupplier, null, result);
            }

            private WorkBucketType getNextBucket(TaskType task) {
                int lastBucketNumber = task.getActivityState() != null ? getLastBucketNumber(task.getActivityState().getActivity().getBucketing().getBucket()) : 0;
                return new WorkBucketType(prismContext).sequentialNumber(lastBucketNumber + 1).state(WorkBucketStateType.DELEGATED).workerRef(String.valueOf(threadIndex), TaskType.COMPLEX_TYPE);
            }

            private int getLastBucketNumber(List<WorkBucketType> buckets) {
                return buckets.stream().mapToInt(WorkBucketType::getSequentialNumber).max().orElse(0);
            }

            @Override
            String description() {
                return "Bucket computer thread #" + threadIndex;
            }
        };
        thread.start();
        threads.add(thread);
    }
    waitForThreads(threads, DURATION);
    PrismObject<TaskType> taskAfter = plainRepositoryService.getObject(TaskType.class, oid, null, result);
    displayValue("task after", taskAfter);
    assertCorrectBucketSequence(taskAfter.asObjectable().getActivityState().getActivity().getBucketing().getBucket());
}

Example 3 with WorkBucketType

use of com.evolveum.midpoint.xml.ns._public.common.common_3.WorkBucketType in project midpoint by Evolveum.

the class TestBucketingStatic method test210GetReleaseCompleteSequenceForWorkers.

/**
 * Check a sequence of get / release / complete operations on coordinator + 5 worker tasks.
 */
@Test
public void test210GetReleaseCompleteSequenceForWorkers() throws Exception {
    OperationResult result = createOperationResult();
    // suspended
    taskAdd(TASK_210_COORDINATOR, result);
    // suspended
    taskAdd(TASK_210_WORKER_1, result);
    // suspended
    taskAdd(TASK_210_WORKER_2, result);
    // suspended
    taskAdd(TASK_210_WORKER_3, result);
    // suspended
    taskAdd(TASK_210_WORKER_4, result);
    // suspended
    taskAdd(TASK_210_WORKER_5, result);
    Task coordinator = taskManager.getTaskPlain(TASK_210_COORDINATOR.oid, result);
    String oidC = TASK_210_COORDINATOR.oid;
    String oidW1 = TASK_210_WORKER_1.oid;
    String oidW2 = TASK_210_WORKER_2.oid;
    String oidW3 = TASK_210_WORKER_3.oid;
    String oidW4 = TASK_210_WORKER_4.oid;
    String oidW5 = TASK_210_WORKER_5.oid;
    when();
    WorkBucketType bucket1 = getWorkBucket(coordinator, oidW1, result);
    WorkBucketType bucket2 = getWorkBucket(coordinator, oidW2, result);
    WorkBucketType bucket3 = getWorkBucket(coordinator, oidW3, result);
    WorkBucketType bucket4 = getWorkBucket(coordinator, oidW4, result);
    // should be the same as bucket4 (the same worker)
    WorkBucketType bucket4a = getWorkBucket(coordinator, oidW4, result);
    then();
    displayValue("1st allocated bucket", bucket1);
    displayValue("2nd allocated bucket", bucket2);
    displayValue("3rd allocated bucket", bucket3);
    displayValue("4th allocated bucket", bucket4);
    displayValue("4+th allocated bucket", bucket4a);
    coordinator.refresh(result);
    displayDumpable("coordinator task after 4+1x allocation", coordinator);
    assertNumericBucket(bucket1, null, 1, 0, 1);
    assertNumericBucket(bucket2, null, 2, 1, 2);
    assertNumericBucket(bucket3, null, 3, 2, 3);
    assertNumericBucket(bucket4, null, 4, 3, 4);
    assertNumericBucket(bucket4a, null, 4, 3, 4);
    List<WorkBucketType> buckets = new ArrayList<>(getBuckets(coordinator));
    sortBucketsBySequentialNumber(buckets);
    assertEquals(4, buckets.size());
    assertNumericBucket(buckets.get(0), WorkBucketStateType.DELEGATED, 1, 0, 1);
    assertNumericBucket(buckets.get(1), WorkBucketStateType.DELEGATED, 2, 1, 2);
    assertNumericBucket(buckets.get(2), WorkBucketStateType.DELEGATED, 3, 2, 3);
    assertNumericBucket(buckets.get(3), WorkBucketStateType.DELEGATED, 4, 3, 4);
    when("complete bucket #2");
    bucketingManager.completeWorkBucket(oidC, oidW2, ActivityPath.empty(), 2, null, null, result);
    then("complete bucket #2");
    coordinator = taskManager.getTaskPlain(coordinator.getOid(), result);
    displayDumpable("coordinator after completion of 2nd bucket", coordinator);
    buckets = new ArrayList<>(getBuckets(coordinator));
    sortBucketsBySequentialNumber(buckets);
    assertEquals(4, buckets.size());
    assertNumericBucket(buckets.get(0), WorkBucketStateType.DELEGATED, 1, 0, 1);
    assertNumericBucket(buckets.get(1), WorkBucketStateType.COMPLETE, 2, 1, 2);
    assertNumericBucket(buckets.get(2), WorkBucketStateType.DELEGATED, 3, 2, 3);
    assertNumericBucket(buckets.get(3), WorkBucketStateType.DELEGATED, 4, 3, 4);
    when("complete bucket #1");
    bucketingManager.completeWorkBucket(oidC, oidW1, ActivityPath.empty(), 1, null, null, result);
    WorkBucketType bucket = getWorkBucket(coordinator, oidW1, result);
    then("complete bucket #1");
    coordinator = taskManager.getTaskPlain(coordinator.getOid(), result);
    displayDumpable("coordinator after completion of 1st bucket and fetching next one", coordinator);
    assertNumericBucket(bucket, null, 5, 4, 5);
    buckets = new ArrayList<>(getBuckets(coordinator));
    sortBucketsBySequentialNumber(buckets);
    assertEquals(4, buckets.size());
    assertNumericBucket(buckets.get(0), WorkBucketStateType.COMPLETE, 2, 1, 2);
    assertNumericBucket(buckets.get(1), WorkBucketStateType.DELEGATED, 3, 2, 3);
    assertNumericBucket(buckets.get(2), WorkBucketStateType.DELEGATED, 4, 3, 4);
    assertNumericBucket(buckets.get(3), WorkBucketStateType.DELEGATED, 5, 4, 5);
    when("no more buckets");
    WorkBucketType nothing = getWorkBucket(coordinator, oidW5, result);
    then("no more buckets");
    assertNull("Found bucket even if none should be found", nothing);
    when("release bucket #4");
    bucketingManager.releaseWorkBucket(oidC, oidW4, ActivityPath.empty(), 4, null, result);
    then("release bucket #4");
    coordinator = taskManager.getTaskPlain(coordinator.getOid(), result);
    displayDumpable("coordinator after releasing of 4th bucket", coordinator);
    buckets = new ArrayList<>(getBuckets(coordinator));
    sortBucketsBySequentialNumber(buckets);
    assertEquals(4, buckets.size());
    assertNumericBucket(buckets.get(0), WorkBucketStateType.COMPLETE, 2, 1, 2);
    assertNumericBucket(buckets.get(1), WorkBucketStateType.DELEGATED, 3, 2, 3);
    assertNumericBucket(buckets.get(2), READY, 4, 3, 4);
    assertNumericBucket(buckets.get(3), WorkBucketStateType.DELEGATED, 5, 4, 5);
    when("complete bucket #3");
    bucketingManager.completeWorkBucket(oidC, oidW3, ActivityPath.empty(), 3, null, null, result);
    bucket = getWorkBucket(coordinator, oidW5, result);
    then("complete bucket #3");
    coordinator = taskManager.getTaskPlain(coordinator.getOid(), result);
    displayDumpable("coordinator after completion of 3rd bucket and getting next one", coordinator);
    assertNumericBucket(bucket, null, 4, 3, 4);
    buckets = new ArrayList<>(getBuckets(coordinator));
    sortBucketsBySequentialNumber(buckets);
    assertEquals(3, buckets.size());
    assertNumericBucket(buckets.get(0), WorkBucketStateType.COMPLETE, 3, 2, 3);
    assertNumericBucket(buckets.get(1), WorkBucketStateType.DELEGATED, 4, 3, 4);
    assertNumericBucket(buckets.get(2), WorkBucketStateType.DELEGATED, 5, 4, 5);
    when("complete bucket #5");
    bucketingManager.completeWorkBucket(oidC, oidW1, ActivityPath.empty(), 5, null, null, result);
    taskManager.closeTask(oidW5, result);
    then("complete bucket #5");
    coordinator = taskManager.getTaskPlain(coordinator.getOid(), result);
    displayDumpable("coordinator after completion of 5th bucket and closing worker5", coordinator);
    buckets = new ArrayList<>(getOrCreateBuckets(coordinator.getWorkState().getActivity()));
    assertEquals(2, buckets.size());
    assertNumericBucket(buckets.get(0), WorkBucketStateType.DELEGATED, 4, 3, 4);
    assertNumericBucket(buckets.get(1), WorkBucketStateType.COMPLETE, 5, 4, 5);
    when("reclaiming mis-allocated bucket");
    bucket = getWorkBucket(coordinator, oidW1, b -> b.withFreeBucketWaitTime(-1).withIsScavenger(true), result);
    assertThat(bucket).isNotNull();
    then("reclaiming mis-allocated bucket");
    coordinator = taskManager.getTaskPlain(coordinator.getOid(), result);
    displayDumpable("coordinator after reclaiming mis-allocated bucket", coordinator);
    assertNumericBucket(bucket, null, 4, 3, 4);
    buckets = new ArrayList<>(getBuckets(coordinator));
    assertEquals(2, buckets.size());
    assertNumericBucket(buckets.get(0), WorkBucketStateType.DELEGATED, 4, 3, 4);
    assertNumericBucket(buckets.get(1), WorkBucketStateType.COMPLETE, 5, 4, 5);
    when("complete bucket #4");
    bucketingManager.completeWorkBucket(oidC, oidW1, ActivityPath.empty(), 4, null, null, result);
    then("complete bucket #4");
    coordinator = taskManager.getTaskPlain(coordinator.getOid(), result);
    displayDumpable("coordinator after completion of 4th bucket", coordinator);
    buckets = new ArrayList<>(getBuckets(coordinator));
    assertEquals(1, buckets.size());
    assertNumericBucket(buckets.get(0), WorkBucketStateType.COMPLETE, 5, 4, 5);
}

Example 4 with WorkBucketType

use of com.evolveum.midpoint.xml.ns._public.common.common_3.WorkBucketType in project midpoint by Evolveum.

the class ReleaseBucketsOperation method computeReleaseModifications.

@NotNull
private Collection<ItemDelta<?, ?>> computeReleaseModifications(@NotNull TaskType task) {
    assert workerTaskOid != null;
    List<WorkBucketType> bucketsToRelease = new ArrayList<>();
    List<WorkBucketType> currentBuckets = BucketingUtil.getBuckets(task.getActivityState(), activityPath);
    if (sequentialNumber != null) {
        WorkBucketType bucket = BucketingUtil.findBucketByNumberRequired(currentBuckets, sequentialNumber);
        checkBucketReadyOrDelegated(bucket);
        bucketsToRelease.add(bucket);
    } else {
        currentBuckets.stream().filter(b -> BucketingUtil.isDelegatedTo(b, workerTaskOid)).forEach(bucketsToRelease::add);
    }
    LOGGER.trace("Releasing buckets {} in {} (delegated to {})", bucketsToRelease, task, workerTaskOid);
    ItemPath statePath = ActivityStateUtil.getStateItemPath(task.getActivityState(), activityPath);
    return bucketsStateChangeDeltas(statePath, bucketsToRelease, WorkBucketStateType.READY, null);
}

Example 5 with WorkBucketType

use of com.evolveum.midpoint.xml.ns._public.common.common_3.WorkBucketType in project midpoint by Evolveum.

the class GetBucketOperationAttempt method offerExistingBuckets.

/**
 * Offers existing buckets for processing:
 *
 * - skipping first N-1 of them,
 * - giving the N-th for use,
 * - not touching the rest.
 *
 * If numberOfBucketsToGet is non-zero at exit, then all existing buckets were consumed.
 */
private void offerExistingBuckets(Stream<WorkBucketType> buckets) {
    Iterator<WorkBucketType> iterator = buckets.iterator();
    while (iterator.hasNext()) {
        WorkBucketType bucket = iterator.next();
        if (numberOfBucketsToGet == 0) {
            // leaving remaining existing buckets intact
            return;
        } else if (numberOfBucketsToGet == 1) {
            markExistingBucketToUse(bucket);
        } else {
            markExistingBucketSkipped(bucket);
        }
        numberOfBucketsToGet--;
    }
}