use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.
the class DynamicPartitionerWithAvroTest method runDynamicPartitionerMR.
private void runDynamicPartitionerMR(final List<? extends GenericRecord> records, boolean allowConcurrentWriters, final boolean precreatePartitions, @Nullable final DynamicPartitioner.PartitionWriteOption partitionWriteOption, boolean expectedStatus) throws Exception {
ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingAvroDynamicPartitioner.class);
final long now = System.currentTimeMillis();
final Multimap<PartitionKey, GenericRecord> keyToRecordsMap = groupByPartitionKey(records, now);
// write values to the input kvTable
final KeyValueTable kvTable = datasetCache.getDataset(INPUT_DATASET);
Transactions.createTransactionExecutor(txExecutorFactory, kvTable).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() {
// the keys are not used; it matters that they're unique though
for (int i = 0; i < records.size(); i++) {
kvTable.write(Integer.toString(i), records.get(i).toString());
}
}
});
final PartitionedFileSet pfs = datasetCache.getDataset(OUTPUT_DATASET);
if (precreatePartitions) {
Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) pfs).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws IOException {
writeFile(pfs, createKey(now, 95111));
writeFile(pfs, createKey(now, 98123));
writeFile(pfs, createKey(now, 84125));
}
});
}
String allowConcurrencyKey = "dataset." + OUTPUT_DATASET + "." + PartitionedFileSetArguments.DYNAMIC_PARTITIONER_ALLOW_CONCURRENCY;
// run the partition writer m/r with this output partition time
Map<String, String> arguments = new HashMap<>();
arguments.put(OUTPUT_PARTITION_KEY, Long.toString(now));
arguments.put(allowConcurrencyKey, Boolean.toString(allowConcurrentWriters));
if (partitionWriteOption != null) {
arguments.put("partitionWriteOption", partitionWriteOption.name());
}
long startTime = System.currentTimeMillis();
boolean status = runProgram(app, AppWithMapReduceUsingAvroDynamicPartitioner.DynamicPartitioningMapReduce.class, new BasicArguments(arguments));
Assert.assertEquals(expectedStatus, status);
if (!expectedStatus) {
// if we expect the program to fail, no need to check the output data for expected results
return;
}
// Verify notifications
List<Notification> notifications = getDataNotifications(startTime);
Assert.assertEquals(1, notifications.size());
Assert.assertEquals(NamespaceId.DEFAULT.dataset(OUTPUT_DATASET), DatasetId.fromString(notifications.get(0).getProperties().get("datasetId")));
// this should have created a partition in the pfs
final Location pfsBaseLocation = pfs.getEmbeddedFileSet().getBaseLocation();
Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) pfs).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() throws IOException {
Map<PartitionKey, PartitionDetail> partitions = new HashMap<>();
for (PartitionDetail partition : pfs.getPartitions(null)) {
partitions.put(partition.getPartitionKey(), partition);
// check that the mapreduce wrote the output partition metadata to all the output partitions
Assert.assertEquals(getExpectedMetadata(precreatePartitions, partitionWriteOption), partition.getMetadata().asMap());
// if files were precreated, and the option is to append, expect the empty file to exist
// if partition write option is configured to overwrite, then the file is expected to not exist
Location preexistingFile = partition.getLocation().append("file");
if (precreatePartitions && partitionWriteOption == DynamicPartitioner.PartitionWriteOption.CREATE_OR_APPEND) {
Assert.assertTrue(preexistingFile.exists());
try (InputStream inputStream = preexistingFile.getInputStream()) {
Assert.assertEquals(-1, inputStream.read());
}
} else {
Assert.assertFalse(preexistingFile.exists());
}
}
Assert.assertEquals(3, partitions.size());
Assert.assertEquals(keyToRecordsMap.keySet(), partitions.keySet());
// Check relative paths of the partitions. Also check that their location = pfs baseLocation + relativePath
for (Map.Entry<PartitionKey, PartitionDetail> partitionKeyEntry : partitions.entrySet()) {
PartitionDetail partitionDetail = partitionKeyEntry.getValue();
String relativePath = partitionDetail.getRelativePath();
int zip = (int) partitionKeyEntry.getKey().getField("zip");
Assert.assertEquals(Long.toString(now) + Path.SEPARATOR + zip, relativePath);
Assert.assertEquals(pfsBaseLocation.append(relativePath), partitionDetail.getLocation());
}
for (Map.Entry<PartitionKey, Collection<GenericRecord>> keyToRecordsEntry : keyToRecordsMap.asMap().entrySet()) {
Set<GenericRecord> genericRecords = new HashSet<>(keyToRecordsEntry.getValue());
Assert.assertEquals(genericRecords, readOutput(partitions.get(keyToRecordsEntry.getKey()).getLocation()));
}
}
});
}
use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.
the class DistributedWorkflowProgramRunnerTest method setupWorkflowRuntime.
/**
* Setup the {@link ProgramLaunchConfig} for the given workflow.
*/
private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, Map<String, String> runtimeArgs) throws IOException {
// Create the distributed workflow program runner
ProgramRunner programRunner = programRunnerFactory.create(ProgramType.WORKFLOW);
Assert.assertTrue(programRunner instanceof DistributedWorkflowProgramRunner);
DistributedWorkflowProgramRunner workflowRunner = (DistributedWorkflowProgramRunner) programRunner;
// Create the Workflow Program
Program workflowProgram = createWorkflowProgram(cConf, programRunner, workflowName);
ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
ProgramOptions programOpts = new SimpleProgramOptions(workflowProgram.getId(), new BasicArguments(), new BasicArguments(runtimeArgs));
// Setup the launching config
workflowRunner.setupLaunchConfig(launchConfig, workflowProgram, programOpts, cConf, new Configuration(), TEMP_FOLDER.newFolder());
return launchConfig;
}
use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.
the class AppFabricTestHelper method submit.
/**
* Submits a program execution.
*
* @param app the application containing the program
* @param programClassName name of the program class
* @param userArgs runtime arguments
* @param folderSupplier a Supplier of temporary folder
* @return a {@link ProgramController} for controlling the program execution.
*/
public static ProgramController submit(ApplicationWithPrograms app, String programClassName, Arguments userArgs, Supplier<File> folderSupplier) throws Exception {
ProgramRunnerFactory runnerFactory = injector.getInstance(ProgramRunnerFactory.class);
ProgramRunner runner = null;
Program program = null;
for (ProgramDescriptor programDescriptor : app.getPrograms()) {
if (programDescriptor.getSpecification().getClassName().equals(programClassName)) {
runner = runnerFactory.create(programDescriptor.getProgramId().getType());
program = createProgram(programDescriptor, app.getArtifactLocation(), runner, folderSupplier);
break;
}
}
Assert.assertNotNull(program);
BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.RUN_ID, RunIds.generate().getId(), ProgramOptionConstants.HOST, InetAddress.getLoopbackAddress().getCanonicalHostName(), ProgramOptionConstants.ARTIFACT_ID, Joiner.on(":").join(app.getArtifactId().toIdParts())));
return runner.run(program, new SimpleProgramOptions(program.getId(), systemArgs, userArgs));
}
use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.
the class AbstractInMemoryProgramRunner method createComponentOptions.
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) {
Map<String, String> systemOptions = Maps.newHashMap();
systemOptions.putAll(options.getArguments().asMap());
systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId));
systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances));
systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId());
systemOptions.put(ProgramOptionConstants.HOST, host);
return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments());
}
use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by cdapio.
the class ProgramNotificationSubscriberServiceTest method testWorkflowInnerPrograms.
@Test
public void testWorkflowInnerPrograms() throws Exception {
AppFabricTestHelper.deployApplication(Id.Namespace.DEFAULT, ProgramStateWorkflowApp.class, null, cConf);
ProgramRunId workflowRunId = NamespaceId.DEFAULT.app(ProgramStateWorkflowApp.class.getSimpleName()).workflow(ProgramStateWorkflowApp.ProgramStateWorkflow.class.getSimpleName()).run(RunIds.generate());
ApplicationSpecification appSpec = TransactionRunners.run(transactionRunner, context -> {
return AppMetadataStore.create(context).getApplication(workflowRunId.getParent().getParent()).getSpec();
});
ProgramDescriptor programDescriptor = new ProgramDescriptor(workflowRunId.getParent(), appSpec);
// Start and run the workflow
Map<String, String> systemArgs = new HashMap<>();
systemArgs.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
systemArgs.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
programStateWriter.start(workflowRunId, new SimpleProgramOptions(workflowRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
programStateWriter.running(workflowRunId, null);
ProgramRunId mrRunId = workflowRunId.getParent().getParent().mr(ProgramStateWorkflowApp.ProgramStateMR.class.getSimpleName()).run(RunIds.generate());
ProgramRunId sparkRunId = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark.class.getSimpleName()).run(RunIds.generate());
ProgramId sparkId2 = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark2.class.getSimpleName());
// Start and run the MR and Spark inside
for (ProgramRunId programRunId : Arrays.asList(mrRunId, sparkRunId)) {
workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.STARTING));
workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.RUNNING));
systemArgs = new HashMap<>(systemArgs);
systemArgs.put(ProgramOptionConstants.RUN_ID, programRunId.getRun());
systemArgs.put(ProgramOptionConstants.WORKFLOW_NAME, workflowRunId.getProgram());
systemArgs.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getRun());
systemArgs.put(ProgramOptionConstants.WORKFLOW_NODE_ID, programRunId.getProgram());
systemArgs.put(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW, programRunId.getProgram());
programStateWriter.start(programRunId, new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
programStateWriter.running(programRunId, null);
// Wait for the inner program running
Tasks.waitFor(ProgramRunStatus.RUNNING, () -> TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
RunRecordDetail meta = metadataStoreDataset.getRun(programRunId);
if (meta == null) {
return null;
}
return meta.getStatus();
}), 10, TimeUnit.SECONDS);
}
// Stop the Spark normally
programStateWriter.completed(sparkRunId);
// Error out the Workflow without stopping the MR
programStateWriter.error(workflowRunId, new IllegalStateException("Explicitly error out"));
// Wait for the Workflow state changed to failed
Tasks.waitFor(ProgramRunStatus.FAILED, () -> TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
RunRecordDetail meta = metadataStoreDataset.getRun(workflowRunId);
if (meta == null) {
return null;
}
return meta.getStatus();
}), 10000, TimeUnit.SECONDS);
// The MR run record should be changed to ERROR state as well (without race)
TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
RunRecordDetail meta = metadataStoreDataset.getRun(mrRunId);
Assert.assertNotNull(meta);
Assert.assertEquals(ProgramRunStatus.FAILED, meta.getStatus());
});
// The Spark run record should stay as COMPLETED
TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
RunRecordDetail meta = metadataStoreDataset.getRun(sparkRunId);
Assert.assertNotNull(meta);
Assert.assertEquals(ProgramRunStatus.COMPLETED, meta.getStatus());
});
// Since the Spark2 program hasn't been executed, there should be no run record
TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
Map<ProgramRunId, RunRecordDetail> runs = metadataStoreDataset.getRuns(sparkId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 100, null);
Assert.assertTrue(runs.isEmpty());
});
}
Aggregations