Examples with ApplicationWithPrograms io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms used on opensource projects

Example 26 with ApplicationWithPrograms

use of io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms in project cdap by caskdata.

the class DynamicPartitionerWithAvroTest method runDynamicPartitionerMR.

private void runDynamicPartitionerMR(final List<? extends GenericRecord> records, boolean allowConcurrentWriters, final boolean precreatePartitions, @Nullable final DynamicPartitioner.PartitionWriteOption partitionWriteOption, boolean expectedStatus) throws Exception {
    ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingAvroDynamicPartitioner.class);
    final long now = System.currentTimeMillis();
    final Multimap<PartitionKey, GenericRecord> keyToRecordsMap = groupByPartitionKey(records, now);
    // write values to the input kvTable
    final KeyValueTable kvTable = datasetCache.getDataset(INPUT_DATASET);
    Transactions.createTransactionExecutor(txExecutorFactory, kvTable).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() {
            // the keys are not used; it matters that they're unique though
            for (int i = 0; i < records.size(); i++) {
                kvTable.write(Integer.toString(i), records.get(i).toString());
            }
        }
    });
    final PartitionedFileSet pfs = datasetCache.getDataset(OUTPUT_DATASET);
    if (precreatePartitions) {
        Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) pfs).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws IOException {
                writeFile(pfs, createKey(now, 95111));
                writeFile(pfs, createKey(now, 98123));
                writeFile(pfs, createKey(now, 84125));
            }
        });
    }
    String allowConcurrencyKey = "dataset." + OUTPUT_DATASET + "." + PartitionedFileSetArguments.DYNAMIC_PARTITIONER_ALLOW_CONCURRENCY;
    // run the partition writer m/r with this output partition time
    Map<String, String> arguments = new HashMap<>();
    arguments.put(OUTPUT_PARTITION_KEY, Long.toString(now));
    arguments.put(allowConcurrencyKey, Boolean.toString(allowConcurrentWriters));
    if (partitionWriteOption != null) {
        arguments.put("partitionWriteOption", partitionWriteOption.name());
    }
    long startTime = System.currentTimeMillis();
    boolean status = runProgram(app, AppWithMapReduceUsingAvroDynamicPartitioner.DynamicPartitioningMapReduce.class, new BasicArguments(arguments));
    Assert.assertEquals(expectedStatus, status);
    if (!expectedStatus) {
        // if we expect the program to fail, no need to check the output data for expected results
        return;
    }
    // Verify notifications
    List<Notification> notifications = getDataNotifications(startTime);
    Assert.assertEquals(1, notifications.size());
    Assert.assertEquals(NamespaceId.DEFAULT.dataset(OUTPUT_DATASET), DatasetId.fromString(notifications.get(0).getProperties().get("datasetId")));
    // this should have created a partition in the pfs
    final Location pfsBaseLocation = pfs.getEmbeddedFileSet().getBaseLocation();
    Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) pfs).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws IOException {
            Map<PartitionKey, PartitionDetail> partitions = new HashMap<>();
            for (PartitionDetail partition : pfs.getPartitions(null)) {
                partitions.put(partition.getPartitionKey(), partition);
                // check that the mapreduce wrote the output partition metadata to all the output partitions
                Assert.assertEquals(getExpectedMetadata(precreatePartitions, partitionWriteOption), partition.getMetadata().asMap());
                // if files were precreated, and the option is to append, expect the empty file to exist
                // if partition write option is configured to overwrite, then the file is expected to not exist
                Location preexistingFile = partition.getLocation().append("file");
                if (precreatePartitions && partitionWriteOption == DynamicPartitioner.PartitionWriteOption.CREATE_OR_APPEND) {
                    Assert.assertTrue(preexistingFile.exists());
                    try (InputStream inputStream = preexistingFile.getInputStream()) {
                        Assert.assertEquals(-1, inputStream.read());
                    }
                } else {
                    Assert.assertFalse(preexistingFile.exists());
                }
            }
            Assert.assertEquals(3, partitions.size());
            Assert.assertEquals(keyToRecordsMap.keySet(), partitions.keySet());
            // Check relative paths of the partitions. Also check that their location = pfs baseLocation + relativePath
            for (Map.Entry<PartitionKey, PartitionDetail> partitionKeyEntry : partitions.entrySet()) {
                PartitionDetail partitionDetail = partitionKeyEntry.getValue();
                String relativePath = partitionDetail.getRelativePath();
                int zip = (int) partitionKeyEntry.getKey().getField("zip");
                Assert.assertEquals(Long.toString(now) + Path.SEPARATOR + zip, relativePath);
                Assert.assertEquals(pfsBaseLocation.append(relativePath), partitionDetail.getLocation());
            }
            for (Map.Entry<PartitionKey, Collection<GenericRecord>> keyToRecordsEntry : keyToRecordsMap.asMap().entrySet()) {
                Set<GenericRecord> genericRecords = new HashSet<>(keyToRecordsEntry.getValue());
                Assert.assertEquals(genericRecords, readOutput(partitions.get(keyToRecordsEntry.getKey()).getLocation()));
            }
        }
    });
}

Example 27 with ApplicationWithPrograms

use of io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms in project cdap by caskdata.

the class WorkflowTest method testOneActionWorkflow.

@Test(timeout = 120 * 1000L)
public void testOneActionWorkflow() throws Exception {
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(OneActionWorkflowApp.class, TEMP_FOLDER_SUPPLIER);
    final Injector injector = AppFabricTestHelper.getInjector();
    final ProgramDescriptor programDescriptor = Iterators.filter(app.getPrograms().iterator(), input -> input.getProgramId().getType() == ProgramType.WORKFLOW).next();
    final SettableFuture<String> completion = SettableFuture.create();
    final ProgramController controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER);
    controller.addListener(new AbstractListener() {

        @Override
        public void init(ProgramController.State currentState, @Nullable Throwable cause) {
            LOG.info("Initializing");
            ArtifactId artifactId = controller.getProgramRunId().getNamespaceId().artifact("test", "1.0").toApiArtifactId();
            setStartAndRunning(injector.getInstance(Store.class), controller.getProgramRunId().getParent(), controller.getProgramRunId().getRun(), artifactId);
        }

        @Override
        public void completed() {
            LOG.info("Completed");
            completion.set("Completed");
        }

        @Override
        public void error(Throwable cause) {
            LOG.info("Error", cause);
            completion.setException(cause);
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    String run = completion.get();
    Assert.assertEquals("Completed", run);
}

Example 28 with ApplicationWithPrograms

use of io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms in project cdap by caskdata.

the class WorkflowTest method testWorkflow.

@Test(timeout = 120 * 1000L)
public void testWorkflow() throws Exception {
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(WorkflowApp.class, TEMP_FOLDER_SUPPLIER);
    final Injector injector = AppFabricTestHelper.getInjector();
    final ProgramDescriptor programDescriptor = Iterators.filter(app.getPrograms().iterator(), input -> input.getProgramId().getType() == ProgramType.WORKFLOW).next();
    String inputPath = createInput();
    String outputPath = new File(tmpFolder.newFolder(), "output").getAbsolutePath();
    BasicArguments userArgs = new BasicArguments(ImmutableMap.of("inputPath", inputPath, "outputPath", outputPath));
    final SettableFuture<String> completion = SettableFuture.create();
    final ProgramController controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), userArgs, TEMP_FOLDER_SUPPLIER);
    controller.addListener(new AbstractListener() {

        @Override
        public void init(ProgramController.State currentState, @Nullable Throwable cause) {
            LOG.info("Starting");
            ArtifactId artifactId = controller.getProgramRunId().getNamespaceId().artifact("test", "1.0").toApiArtifactId();
            setStartAndRunning(injector.getInstance(Store.class), controller.getProgramRunId().getParent(), controller.getProgramRunId().getRun(), artifactId);
        }

        @Override
        public void completed() {
            LOG.info("Completed");
            completion.set("Completed");
        }

        @Override
        public void error(Throwable cause) {
            LOG.info("Error", cause);
            completion.setException(cause);
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    completion.get();
}

Example 29 with ApplicationWithPrograms

use of io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms in project cdap by caskdata.

the class AppLifecycleHttpHandler method deployApplication.

private BodyConsumer deployApplication(final HttpResponder responder, final NamespaceId namespace, final String appId, final String archiveName, final String configString, @Nullable final String ownerPrincipal, final boolean updateSchedules) throws IOException {
    Id.Namespace idNamespace = Id.Namespace.fromEntityId(namespace);
    Location namespaceHomeLocation = namespacePathLocator.get(namespace);
    if (!namespaceHomeLocation.exists()) {
        String msg = String.format("Home directory %s for namespace %s not found", namespaceHomeLocation, namespace.getNamespace());
        LOG.error(msg);
        responder.sendString(HttpResponseStatus.NOT_FOUND, msg);
        return null;
    }
    if (archiveName == null || archiveName.isEmpty()) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, String.format("%s header not present. Please include the header and set its value to the jar name.", ARCHIVE_NAME_HEADER), new DefaultHttpHeaders().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE));
        return null;
    }
    // TODO: (CDAP-3258) error handling needs to be refactored here, should be able just to throw the exception,
    // but the caller catches all exceptions and responds with a 500
    final Id.Artifact artifactId;
    try {
        artifactId = Id.Artifact.parse(idNamespace, archiveName);
    } catch (IllegalArgumentException e) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
        return null;
    }
    KerberosPrincipalId ownerPrincipalId = ownerPrincipal == null ? null : new KerberosPrincipalId(ownerPrincipal);
    // Store uploaded content to a local temp file
    String namespacesDir = configuration.get(Constants.Namespace.NAMESPACES_DIR);
    File localDataDir = new File(configuration.get(Constants.CFG_LOCAL_DATA_DIR));
    File namespaceBase = new File(localDataDir, namespacesDir);
    File tempDir = new File(new File(namespaceBase, namespace.getNamespace()), configuration.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile();
    if (!DirUtils.mkdirs(tempDir)) {
        throw new IOException("Could not create temporary directory at: " + tempDir);
    }
    final KerberosPrincipalId finalOwnerPrincipalId = ownerPrincipalId;
    return new AbstractBodyConsumer(File.createTempFile("app-", ".jar", tempDir)) {

        @Override
        protected void onFinish(HttpResponder responder, File uploadedFile) {
            try {
                // deploy app
                ApplicationWithPrograms app = applicationLifecycleService.deployAppAndArtifact(namespace, appId, artifactId, uploadedFile, configString, finalOwnerPrincipalId, createProgramTerminator(), updateSchedules);
                LOG.info("Successfully deployed app {} in namespace {} from artifact {} with configuration {} and " + "principal {}", app.getApplicationId().getApplication(), namespace.getNamespace(), artifactId, configString, finalOwnerPrincipalId);
                responder.sendString(HttpResponseStatus.OK, String.format("Successfully deployed app %s", app.getApplicationId().getApplication()));
            } catch (InvalidArtifactException e) {
                responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
            } catch (ArtifactAlreadyExistsException e) {
                responder.sendString(HttpResponseStatus.CONFLICT, String.format("Artifact '%s' already exists. Please use the API that creates an application from an existing artifact. " + "If you are trying to replace the artifact, please delete it and then try again.", artifactId));
            } catch (WriteConflictException e) {
                // don't really expect this to happen. It means after multiple retries there were still write conflicts.
                LOG.warn("Write conflict while trying to add artifact {}.", artifactId, e);
                responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Write conflict while adding artifact. This can happen if multiple requests to add " + "the same artifact occur simultaneously. Please try again.");
            } catch (UnauthorizedException e) {
                responder.sendString(HttpResponseStatus.FORBIDDEN, e.getMessage());
            } catch (ConflictException e) {
                responder.sendString(HttpResponseStatus.CONFLICT, e.getMessage());
            } catch (Exception e) {
                LOG.error("Deploy failure", e);
                responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
            }
        }
    };
}

Example 30 with ApplicationWithPrograms

use of io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms in project cdap by caskdata.

the class OpenCloseDataSetTest method testDataSetsAreClosed.

@Test(timeout = 120000)
public void testDataSetsAreClosed() throws Exception {
    final String tableName = "foo";
    TrackingTable.resetTracker();
    ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(DummyAppWithTrackingTable.class, TEMP_FOLDER_SUPPLIER);
    List<ProgramController> controllers = Lists.newArrayList();
    // start the programs
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        if (programDescriptor.getProgramId().getType().equals(ProgramType.MAPREDUCE)) {
            continue;
        }
        // Start service with 1 thread
        Map<String, String> args = Collections.singletonMap(SystemArguments.SERVICE_THREADS, "1");
        controllers.add(AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(args), TEMP_FOLDER_SUPPLIER));
    }
    DiscoveryServiceClient discoveryServiceClient = AppFabricTestHelper.getInjector().getInstance(DiscoveryServiceClient.class);
    Discoverable discoverable = new RandomEndpointStrategy(() -> discoveryServiceClient.discover(String.format("%s.%s.%s.%s", ProgramType.SERVICE.getDiscoverableTypeName(), DefaultId.NAMESPACE.getEntityName(), "dummy", "DummyService"))).pick(5, TimeUnit.SECONDS);
    Assert.assertNotNull(discoverable);
    // write some data to the tracking table through the service
    for (int i = 0; i < 4; i++) {
        String msg = "x" + i;
        URL url = URIScheme.createURI(discoverable, "v3/namespaces/default/apps/dummy/services/DummyService/methods/%s", msg).toURL();
        HttpRequests.execute(HttpRequest.put(url).build(), new DefaultHttpRequestConfig(false));
    }
    // get the number of writes to the foo table
    Assert.assertEquals(4, TrackingTable.getTracker(tableName, "write"));
    // only 2 "open" calls should be tracked:
    // 1. the service has started with one instance (service is loaded lazily on 1st request)
    // 2. DatasetSystemMetadataWriter also instantiates the dataset because it needs to add some system tags
    // for the dataset
    Assert.assertEquals(2, TrackingTable.getTracker(tableName, "open"));
    // now query data from the service
    URL url = URIScheme.createURI(discoverable, "v3/namespaces/default/apps/dummy/services/DummyService/methods/x1").toURL();
    HttpResponse response = HttpRequests.execute(HttpRequest.get(url).build(), new DefaultHttpRequestConfig(false));
    String responseContent = new Gson().fromJson(response.getResponseBodyAsString(), String.class);
    Assert.assertEquals("x1", responseContent);
    // now the dataset must have a read and another open operation
    Assert.assertEquals(1, TrackingTable.getTracker(tableName, "read"));
    // since the same service instance is used, there shouldn't be any new open
    Assert.assertEquals(2, TrackingTable.getTracker(tableName, "open"));
    // The dataset that was instantiated by the DatasetSystemMetadataWriter should have been closed
    Assert.assertEquals(1, TrackingTable.getTracker(tableName, "close"));
    // stop all programs, they should both close the data set foo
    for (ProgramController controller : controllers) {
        controller.stop().get();
    }
    int timesOpened = TrackingTable.getTracker(tableName, "open");
    Assert.assertTrue(timesOpened >= 2);
    Assert.assertEquals(timesOpened, TrackingTable.getTracker(tableName, "close"));
    // now start the m/r job
    ProgramController controller = null;
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        if (programDescriptor.getProgramId().getType().equals(ProgramType.MAPREDUCE)) {
            controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER);
        }
    }
    Assert.assertNotNull(controller);
    while (!controller.getState().equals(ProgramController.State.COMPLETED)) {
        TimeUnit.MILLISECONDS.sleep(100);
    }
    // M/r job is done, one mapper and the m/r client should have opened and closed the data set foo
    // we don't know the exact number of times opened, but it is at least once, and it must be closed the same number
    // of times.
    Assert.assertTrue(timesOpened < TrackingTable.getTracker(tableName, "open"));
    Assert.assertEquals(TrackingTable.getTracker(tableName, "open"), TrackingTable.getTracker(tableName, "close"));
    Assert.assertTrue(0 < TrackingTable.getTracker("bar", "open"));
    Assert.assertEquals(TrackingTable.getTracker("bar", "open"), TrackingTable.getTracker("bar", "close"));
}