use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.
the class AbstractProgramTwillRunnable method createProgramOptions.
/**
* Creates a {@link ProgramOptions} by deserializing the given json file.
*/
private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException {
ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class);
// Overwrite them with environmental information
Map<String, String> arguments = new HashMap<>(original.getArguments().asMap());
arguments.putAll(getExtraSystemArguments());
// Use the name passed in by the constructor as the program name to construct the ProgramId
return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), original.getUserArguments(), original.isDebug());
}
use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.
the class DistributedProgramRunner method updateProgramOptions.
/**
* Creates a new instance of {@link ProgramOptions} with artifact localization information and with
* extra system arguments, while maintaining other fields of the given {@link ProgramOptions}.
*
* @param options the original {@link ProgramOptions}.
* @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container
* @param tempDir a local temporary directory for creating files for artifact localization.
* @param extraSystemArgs a set of extra system arguments to be added/updated
* @return a new instance of {@link ProgramOptions}
* @throws IOException if failed to create local copy of artifact files
*/
private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException {
Arguments systemArgs = options.getArguments();
Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap());
newSystemArgs.putAll(extraSystemArgs);
if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) {
// If the archive already exists locally, we just need to re-localize it to remote containers
File archiveFile = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_ARCHIVE));
// Localize plugins to two files, one expanded into a directory, one not.
localizeResources.put(PLUGIN_DIR, new LocalizeResource(archiveFile, true));
localizeResources.put(PLUGIN_ARCHIVE, new LocalizeResource(archiveFile, false));
} else if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) {
// If there is a plugin directory, then we need to create an archive and localize it to remote containers
File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR));
File archiveFile = new File(tempDir, PLUGIN_DIR + ".jar");
// Store all artifact jars into a new jar file for localization without compression
try (JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(archiveFile))) {
jarOut.setLevel(0);
BundleJarUtil.addToArchive(localDir, jarOut);
}
// Localize plugins to two files, one expanded into a directory, one not.
localizeResources.put(PLUGIN_DIR, new LocalizeResource(archiveFile, true));
localizeResources.put(PLUGIN_ARCHIVE, new LocalizeResource(archiveFile, false));
}
// Add/rename the entries in the system arguments
if (localizeResources.containsKey(PLUGIN_DIR)) {
newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, PLUGIN_DIR);
}
if (localizeResources.containsKey(PLUGIN_ARCHIVE)) {
newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, PLUGIN_ARCHIVE);
}
return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug());
}
use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.
the class AbstractInMemoryProgramRunner method createComponentOptions.
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) {
Map<String, String> systemOptions = Maps.newHashMap();
systemOptions.putAll(options.getArguments().asMap());
systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId));
systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances));
systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId());
systemOptions.put(ProgramOptionConstants.HOST, host);
return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments());
}
use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.
the class MapReduceProgramRunnerTest method testMapReduceWithLocalFiles.
@Test
public void testMapReduceWithLocalFiles() throws Exception {
ApplicationWithPrograms appWithPrograms = deployApp(AppWithLocalFiles.class);
URI stopWordsFile = createStopWordsFile();
final KeyValueTable kvTable = datasetCache.getDataset(AppWithLocalFiles.MR_INPUT_DATASET);
Transactions.createTransactionExecutor(txExecutorFactory, kvTable).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() {
kvTable.write("2324", "a test record");
kvTable.write("43353", "the test table");
kvTable.write("34335", "an end record");
}
});
runProgram(appWithPrograms, AppWithLocalFiles.MapReduceWithLocalFiles.class, new BasicArguments(ImmutableMap.of(AppWithLocalFiles.MR_INPUT_DATASET, "input", AppWithLocalFiles.MR_OUTPUT_DATASET, "output", AppWithLocalFiles.STOPWORDS_FILE_ARG, stopWordsFile.toString())));
final KeyValueTable outputKvTable = datasetCache.getDataset(AppWithLocalFiles.MR_OUTPUT_DATASET);
Transactions.createTransactionExecutor(txExecutorFactory, outputKvTable).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() {
Assert.assertNull(outputKvTable.read("a"));
Assert.assertNull(outputKvTable.read("the"));
Assert.assertNull(outputKvTable.read("an"));
Assert.assertEquals(2, Bytes.toInt(outputKvTable.read("test")));
Assert.assertEquals(2, Bytes.toInt(outputKvTable.read("record")));
Assert.assertEquals(1, Bytes.toInt(outputKvTable.read("table")));
Assert.assertEquals(1, Bytes.toInt(outputKvTable.read("end")));
}
});
}
use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.
the class MapReduceProgramRunnerTest method testTransactionHandling.
/**
* Tests that initialize() and getSplits() are called in the same transaction,
* and with the same instance of the input dataset.
*/
@Test
public void testTransactionHandling() throws Exception {
final ApplicationWithPrograms app = deployApp(AppWithTxAware.class);
runProgram(app, AppWithTxAware.PedanticMapReduce.class, new BasicArguments(ImmutableMap.of("outputPath", TEMP_FOLDER_SUPPLIER.get().getPath() + "/output")));
}
Aggregations