Examples with BasicArguments io.cdap.cdap.internal.app.runtime.BasicArguments used on opensource projects

Example 1 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class AbstractProgramTwillRunnable method createProgramOptions.

/**
 * Creates a {@link ProgramOptions} by deserializing the given json file.
 */
private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException {
    ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class);
    // Overwrite them with environmental information
    Map<String, String> arguments = new HashMap<>(original.getArguments().asMap());
    arguments.putAll(getExtraSystemArguments());
    // Use the name passed in by the constructor as the program name to construct the ProgramId
    return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), original.getUserArguments(), original.isDebug());
}

Example 2 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class DistributedProgramRunner method updateProgramOptions.

/**
 * Creates a new instance of {@link ProgramOptions} with artifact localization information and with
 * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}.
 *
 * @param options the original {@link ProgramOptions}.
 * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container
 * @param tempDir a local temporary directory for creating files for artifact localization.
 * @param extraSystemArgs a set of extra system arguments to be added/updated
 * @return a new instance of {@link ProgramOptions}
 * @throws IOException if failed to create local copy of artifact files
 */
private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException {
    Arguments systemArgs = options.getArguments();
    Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap());
    newSystemArgs.putAll(extraSystemArgs);
    if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) {
        // If the archive already exists locally, we just need to re-localize it to remote containers
        File archiveFile = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_ARCHIVE));
        // Localize plugins to two files, one expanded into a directory, one not.
        localizeResources.put(PLUGIN_DIR, new LocalizeResource(archiveFile, true));
        localizeResources.put(PLUGIN_ARCHIVE, new LocalizeResource(archiveFile, false));
    } else if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) {
        // If there is a plugin directory, then we need to create an archive and localize it to remote containers
        File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR));
        File archiveFile = new File(tempDir, PLUGIN_DIR + ".jar");
        // Store all artifact jars into a new jar file for localization without compression
        try (JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(archiveFile))) {
            jarOut.setLevel(0);
            BundleJarUtil.addToArchive(localDir, jarOut);
        }
        // Localize plugins to two files, one expanded into a directory, one not.
        localizeResources.put(PLUGIN_DIR, new LocalizeResource(archiveFile, true));
        localizeResources.put(PLUGIN_ARCHIVE, new LocalizeResource(archiveFile, false));
    }
    // Add/rename the entries in the system arguments
    if (localizeResources.containsKey(PLUGIN_DIR)) {
        newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, PLUGIN_DIR);
    }
    if (localizeResources.containsKey(PLUGIN_ARCHIVE)) {
        newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, PLUGIN_ARCHIVE);
    }
    return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug());
}

Example 3 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class AbstractInMemoryProgramRunner method createComponentOptions.

private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) {
    Map<String, String> systemOptions = Maps.newHashMap();
    systemOptions.putAll(options.getArguments().asMap());
    systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId));
    systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances));
    systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId());
    systemOptions.put(ProgramOptionConstants.HOST, host);
    return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments());
}

Example 4 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class MapReduceProgramRunnerTest method testMapReduceWithLocalFiles.

@Test
public void testMapReduceWithLocalFiles() throws Exception {
    ApplicationWithPrograms appWithPrograms = deployApp(AppWithLocalFiles.class);
    URI stopWordsFile = createStopWordsFile();
    final KeyValueTable kvTable = datasetCache.getDataset(AppWithLocalFiles.MR_INPUT_DATASET);
    Transactions.createTransactionExecutor(txExecutorFactory, kvTable).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() {
            kvTable.write("2324", "a test record");
            kvTable.write("43353", "the test table");
            kvTable.write("34335", "an end record");
        }
    });
    runProgram(appWithPrograms, AppWithLocalFiles.MapReduceWithLocalFiles.class, new BasicArguments(ImmutableMap.of(AppWithLocalFiles.MR_INPUT_DATASET, "input", AppWithLocalFiles.MR_OUTPUT_DATASET, "output", AppWithLocalFiles.STOPWORDS_FILE_ARG, stopWordsFile.toString())));
    final KeyValueTable outputKvTable = datasetCache.getDataset(AppWithLocalFiles.MR_OUTPUT_DATASET);
    Transactions.createTransactionExecutor(txExecutorFactory, outputKvTable).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() {
            Assert.assertNull(outputKvTable.read("a"));
            Assert.assertNull(outputKvTable.read("the"));
            Assert.assertNull(outputKvTable.read("an"));
            Assert.assertEquals(2, Bytes.toInt(outputKvTable.read("test")));
            Assert.assertEquals(2, Bytes.toInt(outputKvTable.read("record")));
            Assert.assertEquals(1, Bytes.toInt(outputKvTable.read("table")));
            Assert.assertEquals(1, Bytes.toInt(outputKvTable.read("end")));
        }
    });
}

Example 5 with BasicArguments

use of io.cdap.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class MapReduceProgramRunnerTest method testTransactionHandling.

/**
 * Tests that initialize() and getSplits() are called in the same transaction,
 * and with the same instance of the input dataset.
 */
@Test
public void testTransactionHandling() throws Exception {
    final ApplicationWithPrograms app = deployApp(AppWithTxAware.class);
    runProgram(app, AppWithTxAware.PedanticMapReduce.class, new BasicArguments(ImmutableMap.of("outputPath", TEMP_FOLDER_SUPPLIER.get().getPath() + "/output")));
}