use of co.cask.cdap.api.data.DatasetInstantiationException in project cdap by caskdata.
the class MapReduceProgramRunnerTest method testWordCount.
@Test
public void testWordCount() throws Exception {
// deploy to namespace default by default
final ApplicationWithPrograms app = deployApp(AppWithMapReduce.class);
final String inputPath = createInput();
final java.io.File outputDir = new java.io.File(TEMP_FOLDER.newFolder(), "output");
try {
datasetCache.getDataset("someOtherNameSpace", "jobConfig");
Assert.fail("getDataset() should throw an exception when accessing a non-existing dataset.");
} catch (DatasetInstantiationException e) {
// expected
}
// Should work if explicitly specify the default namespace
final KeyValueTable jobConfigTable = datasetCache.getDataset(NamespaceId.DEFAULT.getNamespace(), "jobConfig");
// write config into dataset
Transactions.createTransactionExecutor(txExecutorFactory, jobConfigTable).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() {
jobConfigTable.write(Bytes.toBytes("inputPath"), Bytes.toBytes(inputPath));
jobConfigTable.write(Bytes.toBytes("outputPath"), Bytes.toBytes(outputDir.getPath()));
}
});
runProgram(app, AppWithMapReduce.ClassicWordCount.class, false, true);
Assert.assertEquals("true", System.getProperty("partitioner.initialize"));
Assert.assertEquals("true", System.getProperty("partitioner.destroy"));
Assert.assertEquals("true", System.getProperty("partitioner.set.conf"));
Assert.assertEquals("true", System.getProperty("comparator.initialize"));
Assert.assertEquals("true", System.getProperty("comparator.destroy"));
Assert.assertEquals("true", System.getProperty("comparator.set.conf"));
File[] outputFiles = outputDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith("part-r-") && !name.endsWith(".crc");
}
});
Assert.assertNotNull("no output files found", outputFiles);
int lines = 0;
for (File file : outputFiles) {
lines += Files.readLines(file, Charsets.UTF_8).size();
}
// dummy check that output file is not empty
Assert.assertTrue(lines > 0);
}
use of co.cask.cdap.api.data.DatasetInstantiationException in project cdap by caskdata.
the class MapReduceProgramRunnerTest method testMapreduceWithObjectStore.
@Test
public void testMapreduceWithObjectStore() throws Exception {
// Deploy apps to another namespace and test cross-namespace access meanwhile
final ApplicationWithPrograms app = deployApp(new NamespaceId("someOtherNameSpace").toId(), AppWithMapReduceUsingObjectStore.class);
final ObjectStore<String> input = datasetCache.getDataset("someOtherNameSpace", "keys");
// Get dataset from a non existing namespace
try {
datasetCache.getDataset("nonExistingNameSpace", "keys");
Assert.fail("getDataset() should throw an exception when accessing dataset from a non-existing namespace.");
} catch (DatasetInstantiationException e) {
// expected
}
final String testString = "persisted data";
//Populate some input
Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) input).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() {
input.write(Bytes.toBytes(testString), testString);
input.write(Bytes.toBytes("distributed systems"), "distributed systems");
}
});
runProgram(app, AppWithMapReduceUsingObjectStore.ComputeCounts.class, false, true);
final KeyValueTable output = datasetCache.getDataset("someOtherNameSpace", "count");
//read output and verify result
Transactions.createTransactionExecutor(txExecutorFactory, output).execute(new TransactionExecutor.Subroutine() {
@Override
public void apply() {
byte[] val = output.read(Bytes.toBytes(testString));
Assert.assertTrue(val != null);
Assert.assertEquals(Bytes.toString(val), Integer.toString(testString.length()));
val = output.read(Bytes.toBytes("distributed systems"));
Assert.assertTrue(val != null);
Assert.assertEquals(Bytes.toString(val), "19");
}
});
}
use of co.cask.cdap.api.data.DatasetInstantiationException in project cdap by caskdata.
the class SingleThreadDatasetCache method getDataset.
@Override
public <T extends Dataset> T getDataset(DatasetCacheKey key, boolean bypass) throws DatasetInstantiationException {
Dataset dataset;
try {
if (bypass) {
dataset = datasetLoader.load(key);
} else {
try {
dataset = datasetCache.get(key);
} catch (ExecutionException | UncheckedExecutionException e) {
throw e.getCause();
}
}
} catch (DatasetInstantiationException | ServiceUnavailableException e) {
throw e;
} catch (Throwable t) {
throw new DatasetInstantiationException(String.format("Could not instantiate dataset '%s:%s'", key.getNamespace(), key.getName()), t);
}
// make sure the dataset exists and is of the right type
if (dataset == null) {
throw new DatasetInstantiationException(String.format("Dataset '%s' does not exist", key.getName()));
}
T typedDataset;
try {
@SuppressWarnings("unchecked") T t = (T) dataset;
typedDataset = t;
} catch (Throwable t) {
// must be ClassCastException
throw new DatasetInstantiationException(String.format("Could not cast dataset '%s' to requested type. Actual type is %s.", key.getName(), dataset.getClass().getName()), t);
}
// any transaction aware that is not in the active tx-awares is added to the current tx context (if there is one).
if (!bypass && dataset instanceof TransactionAware) {
TransactionAware txAware = (TransactionAware) dataset;
TransactionAware existing = activeTxAwares.get(key);
if (existing == null) {
activeTxAwares.put(key, txAware);
if (txContext != null) {
txContext.addTransactionAware(txAware);
}
} else if (existing != dataset) {
// this better be the same dataset, otherwise the cache did not work
throw new IllegalStateException(String.format("Unexpected state: Cache returned %s for %s, which is different from the " + "active transaction aware %s for the same key. This should never happen.", dataset, key, existing));
}
}
return typedDataset;
}
Aggregations