Examples with WorkerManager co.cask.cdap.test.WorkerManager used on opensource projects

Example 16 with WorkerManager

use of co.cask.cdap.test.WorkerManager in project cdap by caskdata.

the class MessagingAppTestRun method testWithWorker.

@Test
public void testWithWorker() throws Exception {
    ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
    final WorkerManager workerManager = appManager.getWorkerManager(MessagingApp.MessagingWorker.class.getSimpleName()).start();
    MessagingContext messagingContext = getMessagingContext();
    final MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE);
    // Wait for the worker to create the topic
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            try {
                messagingAdmin.getTopicProperties(MessagingApp.TOPIC);
                return true;
            } catch (TopicNotFoundException e) {
                return false;
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Publish a message
    String message = "message";
    MessagePublisher messagePublisher = messagingContext.getMessagePublisher();
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.TOPIC, message);
    // The worker will publish back a message with payload as concat(message, message)
    final MessageFetcher messageFetcher = messagingContext.getMessageFetcher();
    Tasks.waitFor(message + message, new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                Message message = Iterators.getLast(iterator, null);
                return message == null ? null : message.getPayloadAsString();
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Publish concat(message + message) to the app
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.TOPIC, message + message);
    // timeout.
    try {
        Tasks.waitFor(message + message + message + message, new Callable<String>() {

            @Override
            public String call() throws Exception {
                try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                    Message message = Iterators.getLast(iterator, null);
                    return message == null ? null : message.getPayloadAsString();
                }
            }
        }, 2, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        Assert.fail("Expected timeout exception");
    } catch (TimeoutException e) {
    // expected
    }
    // Now publish a message to the control topic, to unblock the transaction block.
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.CONTROL_TOPIC, message);
    // Should expect a new message as concat(message, message, message, message)
    Tasks.waitFor(message + message + message + message, new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                Message message = Iterators.getLast(iterator, null);
                return message == null ? null : message.getPayloadAsString();
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Wait for the worker to finish and verify that it completes successfully.
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.SECONDS);
}

Example 17 with WorkerManager

use of co.cask.cdap.test.WorkerManager in project cdap by caskdata.

the class MessagingAppTestRun method testTxPublishFetch.

@Test
public void testTxPublishFetch() throws Exception {
    ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
    MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE);
    final WorkerManager workerManager = appManager.getWorkerManager(MessagingApp.TransactionalMessagingWorker.class.getSimpleName());
    // Run the TransactionalMessagingWorker twice, one with getting publisher/fetcher inside TX, one outside.
    for (boolean getInTx : Arrays.asList(true, false)) {
        messagingAdmin.createTopic(MessagingApp.TOPIC);
        workerManager.start(Collections.singletonMap("get.in.tx", Boolean.toString(getInTx)));
        // Wait for the worker to finish and verify that it completes successfully.
        int workerRunCount = getInTx ? 1 : 2;
        workerManager.waitForRuns(ProgramRunStatus.COMPLETED, workerRunCount, 60, TimeUnit.SECONDS);
        messagingAdmin.deleteTopic(MessagingApp.TOPIC);
    }
}

Example 18 with WorkerManager

use of co.cask.cdap.test.WorkerManager in project cdap by caskdata.

the class PartitionCorrectorTestRun method testPartitionCorrector.

@Test
public void testPartitionCorrector() throws Exception {
    ApplicationManager appManager = deployApplication(PartitionExploreCorrectorTestApp.class);
    final int numPartitions = 10;
    addDatasetInstance(TimePartitionedFileSet.class.getName(), "tpfs", PartitionedFileSetProperties.builder().setExploreFormat("csv").setExploreSchema("key int, value string").setEnableExploreOnCreate(true).build());
    DataSetManager<TimePartitionedFileSet> tpfsManager = getDataset("tpfs");
    Date date = DATE_FORMAT.parse("6/4/12 10:00 am");
    long baseTime = date.getTime();
    for (int i = 0; i < numPartitions; i++) {
        createPartition(tpfsManager, baseTime + TimeUnit.MINUTES.toMillis(1) * i, i);
    }
    validateAllPartitions(numPartitions);
    dropAllPartitions();
    validateAllPartitions(0);
    // all partitions are missing. drop/recrete Hive table and add all partitions
    WorkerManager workerManager = appManager.getWorkerManager("PartitionWorker").start(ImmutableMap.of("dataset.name", "tpfs", "batch.size", "5", "verbose", "true"));
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
    validateAllPartitions(numPartitions);
    dropAllPartitions();
    for (int i = numPartitions; i < 2 * numPartitions; i++) {
        createPartition(tpfsManager, baseTime + TimeUnit.MINUTES.toMillis(1) * i, i);
    }
    validateAllPartitions(numPartitions);
    // some partitions are missing, some present keep the Hive table and try to add all partitions
    workerManager = appManager.getWorkerManager("PartitionWorker").start(ImmutableMap.of("dataset.name", "tpfs", "batch.size", "8", "verbose", "false", "disable.explore", "false"));
    workerManager.waitForRuns(ProgramRunStatus.COMPLETED, 2, 60, TimeUnit.SECONDS);
    validateAllPartitions(2 * numPartitions);
}