Examples with BasicArguments co.cask.cdap.internal.app.runtime.BasicArguments used on opensource projects

Example 16 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class AppFabricTestHelper method submit.

/**
   * Submits a program execution.
   *
   * @param app the application containing the program
   * @param programClassName name of the program class
   * @param userArgs runtime arguments
   * @param folderSupplier a Supplier of temporary folder
   * @return a {@link ProgramController} for controlling the program execution.
   */
public static ProgramController submit(ApplicationWithPrograms app, String programClassName, Arguments userArgs, Supplier<File> folderSupplier) throws Exception {
    ProgramRunnerFactory runnerFactory = injector.getInstance(ProgramRunnerFactory.class);
    ProgramRunner runner = null;
    Program program = null;
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        if (programDescriptor.getSpecification().getClassName().equals(programClassName)) {
            runner = runnerFactory.create(programDescriptor.getProgramId().getType());
            program = createProgram(programDescriptor, app.getArtifactLocation(), runner, folderSupplier);
            break;
        }
    }
    Assert.assertNotNull(program);
    BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.RUN_ID, RunIds.generate().getId(), ProgramOptionConstants.HOST, InetAddress.getLoopbackAddress().getCanonicalHostName()));
    return runner.run(program, new SimpleProgramOptions(program.getName(), systemArgs, userArgs));
}

Example 17 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class FlowProgramRunner method createFlowletOptions.

private ProgramOptions createFlowletOptions(String name, int instanceId, int instances, ProgramOptions options) {
    Map<String, String> systemArgs = new HashMap<>();
    systemArgs.putAll(options.getArguments().asMap());
    systemArgs.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId));
    systemArgs.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances));
    return new SimpleProgramOptions(name, new BasicArguments(systemArgs), options.getUserArguments());
}

Example 18 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class MapReduceProgramRunnerTest method testMapReduceWithLocalFiles.

@Test
public void testMapReduceWithLocalFiles() throws Exception {
    ApplicationWithPrograms appWithPrograms = deployApp(AppWithLocalFiles.class);
    URI stopWordsFile = createStopWordsFile();
    final KeyValueTable kvTable = datasetCache.getDataset(AppWithLocalFiles.MR_INPUT_DATASET);
    Transactions.createTransactionExecutor(txExecutorFactory, kvTable).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() {
            kvTable.write("2324", "a test record");
            kvTable.write("43353", "the test table");
            kvTable.write("34335", "an end record");
        }
    });
    runProgram(appWithPrograms, AppWithLocalFiles.MapReduceWithLocalFiles.class, new BasicArguments(ImmutableMap.of(AppWithLocalFiles.MR_INPUT_DATASET, "input", AppWithLocalFiles.MR_OUTPUT_DATASET, "output", AppWithLocalFiles.STOPWORDS_FILE_ARG, stopWordsFile.toString())));
    final KeyValueTable outputKvTable = datasetCache.getDataset(AppWithLocalFiles.MR_OUTPUT_DATASET);
    Transactions.createTransactionExecutor(txExecutorFactory, outputKvTable).execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() {
            Assert.assertNull(outputKvTable.read("a"));
            Assert.assertNull(outputKvTable.read("the"));
            Assert.assertNull(outputKvTable.read("an"));
            Assert.assertEquals(2, Bytes.toInt(outputKvTable.read("test")));
            Assert.assertEquals(2, Bytes.toInt(outputKvTable.read("record")));
            Assert.assertEquals(1, Bytes.toInt(outputKvTable.read("table")));
            Assert.assertEquals(1, Bytes.toInt(outputKvTable.read("end")));
        }
    });
}

Example 19 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class MapReduceProgramRunnerTest method testMapreduceWithFile.

private void testMapreduceWithFile(String inputDatasetName, String inputPaths, String outputDatasetName, String outputPath, Class appClass, Class mrClass, Map<String, String> extraRuntimeArgs, @Nullable final String counterTableName, @Nullable final String outputSeparator) throws Exception {
    final ApplicationWithPrograms app = deployApp(appClass, new AppWithMapReduceUsingFileSet.AppConfig(inputDatasetName, outputDatasetName));
    Map<String, String> runtimeArguments = Maps.newHashMap();
    Map<String, String> inputArgs = Maps.newHashMap();
    Map<String, String> outputArgs = Maps.newHashMap();
    FileSetArguments.setInputPaths(inputArgs, inputPaths);
    FileSetArguments.setOutputPath(outputArgs, outputPath);
    if (outputSeparator != null) {
        outputArgs.put(FileSetProperties.OUTPUT_PROPERTIES_PREFIX + TextOutputFormat.SEPERATOR, "#");
    }
    runtimeArguments.putAll(RuntimeArguments.addScope(Scope.DATASET, inputDatasetName, inputArgs));
    runtimeArguments.putAll(RuntimeArguments.addScope(Scope.DATASET, outputDatasetName, outputArgs));
    if (extraRuntimeArgs != null) {
        runtimeArguments.putAll(extraRuntimeArgs);
    }
    // clear the counters in case a previous test case left behind some values
    if (counterTableName != null) {
        Transactions.execute(datasetCache.newTransactionContext(), "countersVerify", new Runnable() {

            @Override
            public void run() {
                KeyValueTable counters = datasetCache.getDataset(counterTableName);
                counters.delete(AppWithMapReduceUsingRuntimeDatasets.INPUT_RECORDS);
                counters.delete(AppWithMapReduceUsingRuntimeDatasets.REDUCE_KEYS);
            }
        });
    }
    // write a handful of numbers to a file; compute their sum, too.
    final long[] values = { 15L, 17L, 7L, 3L };
    final FileSet input = datasetCache.getDataset(inputDatasetName, inputArgs);
    long sum = 0L, count = 1;
    long inputRecords = 0;
    for (Location inputLocation : input.getInputLocations()) {
        final PrintWriter writer = new PrintWriter(inputLocation.getOutputStream());
        for (long value : values) {
            value *= count;
            writer.println(value);
            sum += value;
            inputRecords++;
        }
        writer.close();
        count++;
    }
    runProgram(app, mrClass, new BasicArguments(runtimeArguments));
    // output location in file system is a directory that contains a part file, a _SUCCESS file, and checksums
    // (.<filename>.crc) for these files. Find the actual part file. Its name begins with "part". In this case,
    // there should be only one part file (with this small data, we have a single reducer).
    final FileSet results = datasetCache.getDataset(outputDatasetName, outputArgs);
    Location resultLocation = results.getOutputLocation();
    if (resultLocation.isDirectory()) {
        for (Location child : resultLocation.list()) {
            if (!child.isDirectory() && child.getName().startsWith("part")) {
                resultLocation = child;
                break;
            }
        }
    }
    Assert.assertFalse(resultLocation.isDirectory());
    // read output and verify result
    String line = CharStreams.readFirstLine(CharStreams.newReaderSupplier(Locations.newInputSupplier(resultLocation), Charsets.UTF_8));
    Assert.assertNotNull(line);
    String[] fields = line.split(outputSeparator == null ? ":" : outputSeparator);
    Assert.assertEquals(2, fields.length);
    Assert.assertEquals(AppWithMapReduceUsingFileSet.FileMapper.ONLY_KEY, fields[0]);
    Assert.assertEquals(sum, Long.parseLong(fields[1]));
    if (counterTableName != null) {
        final long totalInputRecords = inputRecords;
        Transactions.execute(datasetCache.newTransactionContext(), "countersVerify", new Runnable() {

            @Override
            public void run() {
                KeyValueTable counters = datasetCache.getDataset(counterTableName);
                Assert.assertEquals(totalInputRecords, counters.incrementAndGet(AppWithMapReduceUsingRuntimeDatasets.INPUT_RECORDS, 0L));
                Assert.assertEquals(1L, counters.incrementAndGet(AppWithMapReduceUsingRuntimeDatasets.REDUCE_KEYS, 0L));
            }
        });
    }
}

Example 20 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class MapReduceWithMultipleOutputsTest method testMultipleOutputs.

@Test
public void testMultipleOutputs() throws Exception {
    ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingMultipleOutputs.class);
    final FileSet fileSet = datasetCache.getDataset(AppWithMapReduceUsingMultipleOutputs.PURCHASES);
    Location inputFile = fileSet.getBaseLocation().append("inputFile");
    inputFile.createNew();
    PrintWriter writer = new PrintWriter(inputFile.getOutputStream());
    // the PURCHASES dataset consists of purchase records in the format: <customerId> <spend>
    writer.println("1 20");
    writer.println("1 65");
    writer.println("1 30");
    writer.println("2 5");
    writer.println("2 53");
    writer.println("2 45");
    writer.println("3 101");
    writer.close();
    // Using multiple outputs, this MapReduce send the records to a different path of the same dataset, depending
    // on the value in the data (large spend amounts will go to one file, while small will go to another file.
    runProgram(app, AppWithMapReduceUsingMultipleOutputs.SeparatePurchases.class, new BasicArguments());
    FileSet outputFileSet = datasetCache.getDataset(AppWithMapReduceUsingMultipleOutputs.SEPARATED_PURCHASES);
    Assert.assertEquals(ImmutableList.of("1 20", "1 30", "2 5", "2 45"), readFromOutput(outputFileSet, "small_purchases"));
    Assert.assertEquals(ImmutableList.of("1 65", "2 53", "3 101"), readFromOutput(outputFileSet, "large_purchases"));
}