Search in sources :

Example 26 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class DynamicPartitionerWithAvroTest method runDynamicPartitionerMR.

private void runDynamicPartitionerMR(final List<? extends GenericRecord> records, boolean allowConcurrentWriters, final boolean precreatePartitions, @Nullable final DynamicPartitioner.PartitionWriteOption partitionWriteOption, boolean expectedStatus) throws Exception {
    ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingAvroDynamicPartitioner.class);
    final long now = System.currentTimeMillis();
    final Multimap<PartitionKey, GenericRecord> keyToRecordsMap = groupByPartitionKey(records, now);
    // write values to the input kvTable
    final KeyValueTable kvTable = datasetCache.getDataset(INPUT_DATASET);
    Transactions.createTransactionExecutor(txExecutorFactory, kvTable).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() {
            // the keys are not used; it matters that they're unique though
            for (int i = 0; i < records.size(); i++) {
                kvTable.write(Integer.toString(i), records.get(i).toString());
            }
        }
    });
    final PartitionedFileSet pfs = datasetCache.getDataset(OUTPUT_DATASET);
    if (precreatePartitions) {
        Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) pfs).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws IOException {
                writeFile(pfs, createKey(now, 95111));
                writeFile(pfs, createKey(now, 98123));
                writeFile(pfs, createKey(now, 84125));
            }
        });
    }
    String allowConcurrencyKey = "dataset." + OUTPUT_DATASET + "." + PartitionedFileSetArguments.DYNAMIC_PARTITIONER_ALLOW_CONCURRENCY;
    // run the partition writer m/r with this output partition time
    Map<String, String> arguments = new HashMap<>();
    arguments.put(OUTPUT_PARTITION_KEY, Long.toString(now));
    arguments.put(allowConcurrencyKey, Boolean.toString(allowConcurrentWriters));
    if (partitionWriteOption != null) {
        arguments.put("partitionWriteOption", partitionWriteOption.name());
    }
    long startTime = System.currentTimeMillis();
    boolean status = runProgram(app, AppWithMapReduceUsingAvroDynamicPartitioner.DynamicPartitioningMapReduce.class, new BasicArguments(arguments));
    Assert.assertEquals(expectedStatus, status);
    if (!expectedStatus) {
        // if we expect the program to fail, no need to check the output data for expected results
        return;
    }
    // Verify notifications
    List<Notification> notifications = getDataNotifications(startTime);
    Assert.assertEquals(1, notifications.size());
    Assert.assertEquals(NamespaceId.DEFAULT.dataset(OUTPUT_DATASET), DatasetId.fromString(notifications.get(0).getProperties().get("datasetId")));
    // this should have created a partition in the pfs
    final Location pfsBaseLocation = pfs.getEmbeddedFileSet().getBaseLocation();
    Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) pfs).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws IOException {
            Map<PartitionKey, PartitionDetail> partitions = new HashMap<>();
            for (PartitionDetail partition : pfs.getPartitions(null)) {
                partitions.put(partition.getPartitionKey(), partition);
                // check that the mapreduce wrote the output partition metadata to all the output partitions
                Assert.assertEquals(getExpectedMetadata(precreatePartitions, partitionWriteOption), partition.getMetadata().asMap());
                // if files were precreated, and the option is to append, expect the empty file to exist
                // if partition write option is configured to overwrite, then the file is expected to not exist
                Location preexistingFile = partition.getLocation().append("file");
                if (precreatePartitions && partitionWriteOption == DynamicPartitioner.PartitionWriteOption.CREATE_OR_APPEND) {
                    Assert.assertTrue(preexistingFile.exists());
                    try (InputStream inputStream = preexistingFile.getInputStream()) {
                        Assert.assertEquals(-1, inputStream.read());
                    }
                } else {
                    Assert.assertFalse(preexistingFile.exists());
                }
            }
            Assert.assertEquals(3, partitions.size());
            Assert.assertEquals(keyToRecordsMap.keySet(), partitions.keySet());
            // Check relative paths of the partitions. Also check that their location = pfs baseLocation + relativePath
            for (Map.Entry<PartitionKey, PartitionDetail> partitionKeyEntry : partitions.entrySet()) {
                PartitionDetail partitionDetail = partitionKeyEntry.getValue();
                String relativePath = partitionDetail.getRelativePath();
                int zip = (int) partitionKeyEntry.getKey().getField("zip");
                Assert.assertEquals(Long.toString(now) + Path.SEPARATOR + zip, relativePath);
                Assert.assertEquals(pfsBaseLocation.append(relativePath), partitionDetail.getLocation());
            }
            for (Map.Entry<PartitionKey, Collection<GenericRecord>> keyToRecordsEntry : keyToRecordsMap.asMap().entrySet()) {
                Set<GenericRecord> genericRecords = new HashSet<>(keyToRecordsEntry.getValue());
                Assert.assertEquals(genericRecords, readOutput(partitions.get(keyToRecordsEntry.getKey()).getLocation()));
            }
        }
    });
}
Also used : HashSet(java.util.HashSet) PartitionedFileSet(io.cdap.cdap.api.dataset.lib.PartitionedFileSet) Set(java.util.Set) HashMap(java.util.HashMap) PartitionDetail(io.cdap.cdap.api.dataset.lib.PartitionDetail) Notification(io.cdap.cdap.proto.Notification) ApplicationWithPrograms(io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) GenericRecord(org.apache.avro.generic.GenericRecord) InputStream(java.io.InputStream) TransactionExecutor(org.apache.tephra.TransactionExecutor) PartitionedFileSet(io.cdap.cdap.api.dataset.lib.PartitionedFileSet) IOException(java.io.IOException) KeyValueTable(io.cdap.cdap.api.dataset.lib.KeyValueTable) TransactionAware(org.apache.tephra.TransactionAware) PartitionKey(io.cdap.cdap.api.dataset.lib.PartitionKey) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) Location(org.apache.twill.filesystem.Location)

Example 27 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class DistributedWorkflowProgramRunnerTest method setupWorkflowRuntime.

/**
 * Setup the {@link ProgramLaunchConfig} for the given workflow.
 */
private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, Map<String, String> runtimeArgs) throws IOException {
    // Create the distributed workflow program runner
    ProgramRunner programRunner = programRunnerFactory.create(ProgramType.WORKFLOW);
    Assert.assertTrue(programRunner instanceof DistributedWorkflowProgramRunner);
    DistributedWorkflowProgramRunner workflowRunner = (DistributedWorkflowProgramRunner) programRunner;
    // Create the Workflow Program
    Program workflowProgram = createWorkflowProgram(cConf, programRunner, workflowName);
    ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
    ProgramOptions programOpts = new SimpleProgramOptions(workflowProgram.getId(), new BasicArguments(), new BasicArguments(runtimeArgs));
    // Setup the launching config
    workflowRunner.setupLaunchConfig(launchConfig, workflowProgram, programOpts, cConf, new Configuration(), TEMP_FOLDER.newFolder());
    return launchConfig;
}
Also used : Program(io.cdap.cdap.app.program.Program) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) ProgramRunner(io.cdap.cdap.app.runtime.ProgramRunner) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions)

Example 28 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class AppFabricTestHelper method submit.

/**
 * Submits a program execution.
 *
 * @param app the application containing the program
 * @param programClassName name of the program class
 * @param userArgs runtime arguments
 * @param folderSupplier a Supplier of temporary folder
 * @return a {@link ProgramController} for controlling the program execution.
 */
public static ProgramController submit(ApplicationWithPrograms app, String programClassName, Arguments userArgs, Supplier<File> folderSupplier) throws Exception {
    ProgramRunnerFactory runnerFactory = injector.getInstance(ProgramRunnerFactory.class);
    ProgramRunner runner = null;
    Program program = null;
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        if (programDescriptor.getSpecification().getClassName().equals(programClassName)) {
            runner = runnerFactory.create(programDescriptor.getProgramId().getType());
            program = createProgram(programDescriptor, app.getArtifactLocation(), runner, folderSupplier);
            break;
        }
    }
    Assert.assertNotNull(program);
    BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.RUN_ID, RunIds.generate().getId(), ProgramOptionConstants.HOST, InetAddress.getLoopbackAddress().getCanonicalHostName(), ProgramOptionConstants.ARTIFACT_ID, Joiner.on(":").join(app.getArtifactId().toIdParts())));
    return runner.run(program, new SimpleProgramOptions(program.getId(), systemArgs, userArgs));
}
Also used : Program(io.cdap.cdap.app.program.Program) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) ProgramRunner(io.cdap.cdap.app.runtime.ProgramRunner) ProgramRunnerFactory(io.cdap.cdap.app.runtime.ProgramRunnerFactory)

Example 29 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class AbstractInMemoryProgramRunner method createComponentOptions.

private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) {
    Map<String, String> systemOptions = Maps.newHashMap();
    systemOptions.putAll(options.getArguments().asMap());
    systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId));
    systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances));
    systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId());
    systemOptions.put(ProgramOptionConstants.HOST, host);
    return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments());
}
Also used : SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments)

Example 30 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.

the class ProgramNotificationSubscriberServiceTest method testWorkflowInnerPrograms.

@Test
public void testWorkflowInnerPrograms() throws Exception {
    AppFabricTestHelper.deployApplication(Id.Namespace.DEFAULT, ProgramStateWorkflowApp.class, null, cConf);
    ProgramRunId workflowRunId = NamespaceId.DEFAULT.app(ProgramStateWorkflowApp.class.getSimpleName()).workflow(ProgramStateWorkflowApp.ProgramStateWorkflow.class.getSimpleName()).run(RunIds.generate());
    ApplicationSpecification appSpec = TransactionRunners.run(transactionRunner, context -> {
        return AppMetadataStore.create(context).getApplication(workflowRunId.getParent().getParent()).getSpec();
    });
    ProgramDescriptor programDescriptor = new ProgramDescriptor(workflowRunId.getParent(), appSpec);
    // Start and run the workflow
    Map<String, String> systemArgs = new HashMap<>();
    systemArgs.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    systemArgs.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    programStateWriter.start(workflowRunId, new SimpleProgramOptions(workflowRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
    programStateWriter.running(workflowRunId, null);
    ProgramRunId mrRunId = workflowRunId.getParent().getParent().mr(ProgramStateWorkflowApp.ProgramStateMR.class.getSimpleName()).run(RunIds.generate());
    ProgramRunId sparkRunId = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark.class.getSimpleName()).run(RunIds.generate());
    ProgramId sparkId2 = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark2.class.getSimpleName());
    // Start and run the MR and Spark inside
    for (ProgramRunId programRunId : Arrays.asList(mrRunId, sparkRunId)) {
        workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.STARTING));
        workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.RUNNING));
        systemArgs = new HashMap<>(systemArgs);
        systemArgs.put(ProgramOptionConstants.RUN_ID, programRunId.getRun());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_NAME, workflowRunId.getProgram());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getRun());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_NODE_ID, programRunId.getProgram());
        systemArgs.put(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW, programRunId.getProgram());
        programStateWriter.start(programRunId, new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
        programStateWriter.running(programRunId, null);
        // Wait for the inner program running
        Tasks.waitFor(ProgramRunStatus.RUNNING, () -> TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
            RunRecordDetail meta = metadataStoreDataset.getRun(programRunId);
            if (meta == null) {
                return null;
            }
            return meta.getStatus();
        }), 10, TimeUnit.SECONDS);
    }
    // Stop the Spark normally
    programStateWriter.completed(sparkRunId);
    // Error out the Workflow without stopping the MR
    programStateWriter.error(workflowRunId, new IllegalStateException("Explicitly error out"));
    // Wait for the Workflow state changed to failed
    Tasks.waitFor(ProgramRunStatus.FAILED, () -> TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(workflowRunId);
        if (meta == null) {
            return null;
        }
        return meta.getStatus();
    }), 10000, TimeUnit.SECONDS);
    // The MR run record should be changed to ERROR state as well (without race)
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(mrRunId);
        Assert.assertNotNull(meta);
        Assert.assertEquals(ProgramRunStatus.FAILED, meta.getStatus());
    });
    // The Spark run record should stay as COMPLETED
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(sparkRunId);
        Assert.assertNotNull(meta);
        Assert.assertEquals(ProgramRunStatus.COMPLETED, meta.getStatus());
    });
    // Since the Spark2 program hasn't been executed, there should be no run record
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        Map<ProgramRunId, RunRecordDetail> runs = metadataStoreDataset.getRuns(sparkId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 100, null);
        Assert.assertTrue(runs.isEmpty());
    });
}
Also used : RunRecordDetail(io.cdap.cdap.internal.app.store.RunRecordDetail) Arrays(java.util.Arrays) TransactionRunners(io.cdap.cdap.spi.data.transaction.TransactionRunners) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) TimeoutException(java.util.concurrent.TimeoutException) NodeStatus(io.cdap.cdap.api.workflow.NodeStatus) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) AppFabricTestHelper(io.cdap.cdap.internal.AppFabricTestHelper) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) After(org.junit.After) Map(java.util.Map) RunId(org.apache.twill.api.RunId) Tasks(io.cdap.cdap.common.utils.Tasks) AfterClass(org.junit.AfterClass) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ProgramRunStatus(io.cdap.cdap.proto.ProgramRunStatus) Id(io.cdap.cdap.common.id.Id) List(java.util.List) AggregationFunction(io.cdap.cdap.api.dataset.lib.cube.AggregationFunction) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) Constants(io.cdap.cdap.common.conf.Constants) ProfileId(io.cdap.cdap.proto.id.ProfileId) ProgramOptionConstants(io.cdap.cdap.internal.app.runtime.ProgramOptionConstants) BeforeClass(org.junit.BeforeClass) MetricStore(io.cdap.cdap.api.metrics.MetricStore) HashMap(java.util.HashMap) ProgramType(io.cdap.cdap.proto.ProgramType) ArrayList(java.util.ArrayList) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ProgramHeartbeatTable(io.cdap.cdap.reporting.ProgramHeartbeatTable) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) Profile(io.cdap.cdap.proto.profile.Profile) MetricDataQuery(io.cdap.cdap.api.metrics.MetricDataQuery) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) WorkflowNodeStateDetail(io.cdap.cdap.proto.WorkflowNodeStateDetail) AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) DefaultApplicationSpecification(io.cdap.cdap.internal.app.DefaultApplicationSpecification) WorkflowStateWriter(io.cdap.cdap.internal.app.runtime.workflow.WorkflowStateWriter) ProfileService(io.cdap.cdap.internal.profile.ProfileService) RunIds(io.cdap.cdap.common.app.RunIds) ProgramId(io.cdap.cdap.proto.id.ProgramId) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) Test(org.junit.Test) MetricTimeSeries(io.cdap.cdap.api.metrics.MetricTimeSeries) ProjectInfo(io.cdap.cdap.common.utils.ProjectInfo) ProgramRunClusterStatus(io.cdap.cdap.proto.ProgramRunClusterStatus) Injector(com.google.inject.Injector) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) TimeValue(io.cdap.cdap.api.dataset.lib.cube.TimeValue) Assert(org.junit.Assert) Collections(java.util.Collections) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) DefaultApplicationSpecification(io.cdap.cdap.internal.app.DefaultApplicationSpecification) AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) HashMap(java.util.HashMap) RunRecordDetail(io.cdap.cdap.internal.app.store.RunRecordDetail) ProgramId(io.cdap.cdap.proto.id.ProgramId) WorkflowNodeStateDetail(io.cdap.cdap.proto.WorkflowNodeStateDetail) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) Test(org.junit.Test)

Aggregations

BasicArguments (io.cdap.cdap.internal.app.runtime.BasicArguments)90 SimpleProgramOptions (io.cdap.cdap.internal.app.runtime.SimpleProgramOptions)54 Test (org.junit.Test)44 ProgramOptions (io.cdap.cdap.app.runtime.ProgramOptions)36 ProgramDescriptor (io.cdap.cdap.app.program.ProgramDescriptor)32 ApplicationWithPrograms (io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms)32 HashMap (java.util.HashMap)32 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)28 ProgramId (io.cdap.cdap.proto.id.ProgramId)24 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)22 SystemArguments (io.cdap.cdap.internal.app.runtime.SystemArguments)20 ImmutableMap (com.google.common.collect.ImmutableMap)18 Map (java.util.Map)18 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)16 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)16 IOException (java.io.IOException)16 Injector (com.google.inject.Injector)14 Collections (java.util.Collections)14 ProgramController (io.cdap.cdap.app.runtime.ProgramController)12 RunIds (io.cdap.cdap.common.app.RunIds)12