Examples with TaggedResource io.pravega.controller.store.task.TaggedResource used on opensource projects

Example 1 with TaggedResource

use of io.pravega.controller.store.task.TaggedResource in project pravega by pravega.

the class TaskTest method testTaskSweeper.

@Test
public void testTaskSweeper() throws ExecutionException, InterruptedException {
    final String deadHost = "deadHost";
    final String deadThreadId = UUID.randomUUID().toString();
    final String scope = SCOPE;
    final String stream = "streamSweeper";
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final Resource resource = new Resource(scope, stream);
    final long timestamp = System.currentTimeMillis();
    final TaskData taskData = new TaskData("createStream", "1.0", new Serializable[] { scope, stream, configuration, timestamp });
    for (int i = 0; i < 5; i++) {
        final TaggedResource taggedResource = new TaggedResource(UUID.randomUUID().toString(), resource);
        taskMetadataStore.putChild(deadHost, taggedResource).join();
    }
    final TaggedResource taggedResource = new TaggedResource(deadThreadId, resource);
    taskMetadataStore.putChild(deadHost, taggedResource).join();
    taskMetadataStore.lock(resource, taskData, deadHost, deadThreadId, null, null).join();
    TaskSweeper taskSweeper = new TaskSweeper(taskMetadataStore, HOSTNAME, executor, streamMetadataTasks);
    taskSweeper.handleFailedProcess(deadHost).get();
    Optional<TaskData> data = taskMetadataStore.getTask(resource, deadHost, deadThreadId).get();
    assertFalse(data.isPresent());
    Optional<TaggedResource> child = taskMetadataStore.getRandomChild(deadHost).get();
    assertFalse(child.isPresent());
    // ensure that the stream streamSweeper is created
    StreamConfiguration config = streamStore.getConfiguration(SCOPE, stream, null, executor).get();
    assertTrue(config.getStreamName().equals(configuration.getStreamName()));
    assertTrue(config.getScope().equals(configuration.getScope()));
    assertTrue(config.getScalingPolicy().equals(configuration.getScalingPolicy()));
}

Example 2 with TaggedResource

use of io.pravega.controller.store.task.TaggedResource in project pravega by pravega.

the class TaskTest method parallelTaskSweeperTest.

@Test
public void parallelTaskSweeperTest() throws InterruptedException, ExecutionException {
    final String deadHost = "deadHost";
    final String deadThreadId1 = UUID.randomUUID().toString();
    final String deadThreadId2 = UUID.randomUUID().toString();
    final String scope = SCOPE;
    final String stream1 = "parallelSweeper1";
    final String stream2 = "parallelSweeper2";
    final StreamConfiguration config1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final StreamConfiguration config2 = StreamConfiguration.builder().scope(SCOPE).streamName(stream2).scalingPolicy(policy1).build();
    final Resource resource1 = new Resource(scope, stream1);
    final long timestamp1 = System.currentTimeMillis();
    final TaskData taskData1 = new TaskData("createStream", "1.0", new Serializable[] { scope, stream1, config1, timestamp1 });
    final Resource resource2 = new Resource(scope, stream2);
    final long timestamp2 = System.currentTimeMillis();
    final TaskData taskData2 = new TaskData("createStream", "1.0", new Serializable[] { scope, stream2, config2, timestamp2 });
    for (int i = 0; i < 5; i++) {
        final TaggedResource taggedResource = new TaggedResource(UUID.randomUUID().toString(), resource1);
        taskMetadataStore.putChild(deadHost, taggedResource).join();
    }
    final TaggedResource taggedResource1 = new TaggedResource(deadThreadId1, resource1);
    taskMetadataStore.putChild(deadHost, taggedResource1).join();
    final TaggedResource taggedResource2 = new TaggedResource(deadThreadId2, resource2);
    taskMetadataStore.putChild(deadHost, taggedResource2).join();
    taskMetadataStore.lock(resource1, taskData1, deadHost, deadThreadId1, null, null).join();
    taskMetadataStore.lock(resource2, taskData2, deadHost, deadThreadId2, null, null).join();
    final SweeperThread sweeperThread1 = new SweeperThread(HOSTNAME, executor, taskMetadataStore, streamMetadataTasks, deadHost);
    final SweeperThread sweeperThread2 = new SweeperThread(HOSTNAME, executor, taskMetadataStore, streamMetadataTasks, deadHost);
    sweeperThread1.start();
    sweeperThread2.start();
    sweeperThread1.getResult().join();
    sweeperThread2.getResult().join();
    Optional<TaskData> data = taskMetadataStore.getTask(resource1, deadHost, deadThreadId1).get();
    assertFalse(data.isPresent());
    data = taskMetadataStore.getTask(resource2, deadHost, deadThreadId2).get();
    assertFalse(data.isPresent());
    Optional<TaggedResource> child = taskMetadataStore.getRandomChild(deadHost).get();
    assertFalse(child.isPresent());
    // ensure that the stream streamSweeper is created
    StreamConfiguration config = streamStore.getConfiguration(SCOPE, stream1, null, executor).get();
    assertTrue(config.getStreamName().equals(stream1));
    config = streamStore.getConfiguration(SCOPE, stream2, null, executor).get();
    assertTrue(config.getStreamName().equals(stream2));
}

Example 3 with TaggedResource

use of io.pravega.controller.store.task.TaggedResource in project pravega by pravega.

the class TaskSweeper method executeHostTask.

private CompletableFuture<Result> executeHostTask(final String oldHostId) {
    // Get a random child TaggedResource of oldHostId node and attempt to execute corresponding task
    return taskMetadataStore.getRandomChild(oldHostId).thenComposeAsync(taggedResourceOption -> {
        if (!taggedResourceOption.isPresent()) {
            log.debug("Host={} fetched no child of {}", this.hostId, oldHostId);
            // Moreover, no need to get any more children, hence return null.
            return taskMetadataStore.removeNode(oldHostId).thenApplyAsync(x -> null, executor);
        } else {
            TaggedResource taggedResource = taggedResourceOption.get();
            log.debug("Host={} processing child <{}, {}> of {}", this.hostId, taggedResource.getResource(), taggedResource.getTag(), oldHostId);
            // and compete to execute it to completion.
            return executeResourceTask(oldHostId, taggedResource);
        }
    }, executor);
}

Example 4 with TaggedResource

use of io.pravega.controller.store.task.TaggedResource in project pravega by pravega.

the class TaskMetadataStoreTests method testFolderOperations.

@Test(timeout = 10000)
public void testFolderOperations() throws ExecutionException, InterruptedException {
    final TaggedResource child1 = new TaggedResource(UUID.randomUUID().toString(), resource);
    final TaggedResource child2 = new TaggedResource(UUID.randomUUID().toString(), resource);
    final TaggedResource child3 = new TaggedResource(UUID.randomUUID().toString(), resource);
    Set<String> hosts = taskMetadataStore.getHosts().get();
    assertTrue(hosts.isEmpty());
    taskMetadataStore.putChild(host1, child1).get();
    taskMetadataStore.putChild(host1, child2).get();
    Optional<TaggedResource> child = taskMetadataStore.getRandomChild(host1).get();
    assertTrue(child.isPresent());
    assertTrue(child.get().getResource().equals(resource));
    taskMetadataStore.removeChild(host1, child1, true).get();
    child = taskMetadataStore.getRandomChild(host1).get();
    assertTrue(child.isPresent());
    assertTrue(child.get().getResource().equals(resource));
    taskMetadataStore.removeChild(host1, child3, true).get();
    hosts = taskMetadataStore.getHosts().get();
    assertEquals(1, hosts.size());
    child = taskMetadataStore.getRandomChild(host1).get();
    assertTrue(child.isPresent());
    assertTrue(child.get().getResource().equals(resource));
    taskMetadataStore.removeChild(host1, child2, true).get();
    child = taskMetadataStore.getRandomChild(host1).get();
    assertFalse(child.isPresent());
}

Example 5 with TaggedResource

use of io.pravega.controller.store.task.TaggedResource in project pravega by pravega.

the class TaskBase method execute.

/**
 * Wrapper method that initially obtains lock then executes the passed method, and finally releases lock.
 *
 * @param resource resource to be updated by the task.
 * @param parameters method parameters.
 * @param operation lambda operation that is the actual task.
 * @param <T> type parameter of return value of operation to be executed.
 * @return return value of task execution.
 */
public <T> CompletableFuture<T> execute(final Resource resource, final Serializable[] parameters, final FutureOperation<T> operation) {
    if (!ready) {
        return Futures.failedFuture(new IllegalStateException(getClass().getName() + " not yet ready"));
    }
    final String tag = UUID.randomUUID().toString();
    final TaskData taskData = getTaskData(parameters);
    final CompletableFuture<T> result = new CompletableFuture<>();
    final TaggedResource taggedResource = new TaggedResource(tag, resource);
    log.debug("Host={}, Tag={} starting to execute task {}-{} on resource {}", context.hostId, tag, taskData.getMethodName(), taskData.getMethodVersion(), resource);
    if (createIndexOnlyMode) {
        return createIndexes(taggedResource, taskData);
    }
    // PutChild (HostId, resource)
    // Initially store the fact that I am about the update the resource.
    // Since multiple threads within this process could concurrently attempt to modify same resource,
    // we tag the resource name with a random GUID so as not to interfere with other thread's
    // creation or deletion of resource children under HostId node.
    taskMetadataStore.putChild(context.hostId, taggedResource).thenComposeAsync(x -> executeTask(resource, taskData, tag, operation), executor).whenCompleteAsync((value, e) -> taskMetadataStore.removeChild(context.hostId, taggedResource, true).whenCompleteAsync((innerValue, innerE) -> {
        // ignore the result of removeChile operations, since it is an optimization
        if (e != null) {
            result.completeExceptionally(e);
        } else {
            result.complete(value);
        }
    }, executor), executor);
    return result;
}