Examples with Dataset io.cdap.cdap.api.dataset.Dataset used on opensource projects

Example 11 with Dataset

use of io.cdap.cdap.api.dataset.Dataset in project cdap by caskdata.

the class BasicMapReduceTaskContext method getBatchWritable.

/**
 * Returns a {@link CloseableBatchWritable} that writes data to the given dataset.
 */
<K, V> CloseableBatchWritable<K, V> getBatchWritable(String namespace, String datasetName, Map<String, String> datasetArgs) {
    Dataset dataset = getDataset(namespace, datasetName, datasetArgs, AccessType.WRITE);
    // Must be BatchWritable.
    Preconditions.checkArgument(dataset instanceof BatchWritable, "Dataset '%s:%s' is not a BatchWritable.", namespace, datasetName);
    @SuppressWarnings("unchecked") final BatchWritable<K, V> delegate = (BatchWritable<K, V>) dataset;
    return new CloseableBatchWritable<K, V>() {

        @Override
        public void write(K k, V v) {
            delegate.write(k, v);
        }

        @Override
        public void close() throws IOException {
            try {
                flushOperations();
            } catch (Exception e) {
                Throwables.propagateIfInstanceOf(e, IOException.class);
                throw new IOException(e);
            }
        }
    };
}

Example 12 with Dataset

use of io.cdap.cdap.api.dataset.Dataset in project cdap by cdapio.

the class BasicMapReduceTaskContext method getBatchWritable.

/**
 * Returns a {@link CloseableBatchWritable} that writes data to the given dataset.
 */
<K, V> CloseableBatchWritable<K, V> getBatchWritable(String namespace, String datasetName, Map<String, String> datasetArgs) {
    Dataset dataset = getDataset(namespace, datasetName, datasetArgs, AccessType.WRITE);
    // Must be BatchWritable.
    Preconditions.checkArgument(dataset instanceof BatchWritable, "Dataset '%s:%s' is not a BatchWritable.", namespace, datasetName);
    @SuppressWarnings("unchecked") final BatchWritable<K, V> delegate = (BatchWritable<K, V>) dataset;
    return new CloseableBatchWritable<K, V>() {

        @Override
        public void write(K k, V v) {
            delegate.write(k, v);
        }

        @Override
        public void close() throws IOException {
            try {
                flushOperations();
            } catch (Exception e) {
                Throwables.propagateIfInstanceOf(e, IOException.class);
                throw new IOException(e);
            }
        }
    };
}

Example 13 with Dataset

use of io.cdap.cdap.api.dataset.Dataset in project cdap by cdapio.

the class BasicMapReduceTaskContext method getBatchReadable.

/**
 * Returns a {@link BatchReadable} that reads data from the given dataset.
 */
<K, V> BatchReadable<K, V> getBatchReadable(@Nullable String datasetNamespace, String datasetName, Map<String, String> datasetArgs) {
    Dataset dataset;
    if (datasetNamespace == null) {
        dataset = getDataset(datasetName, datasetArgs, AccessType.READ);
    } else {
        dataset = getDataset(datasetNamespace, datasetName, datasetArgs, AccessType.READ);
    }
    // Must be BatchReadable.
    Preconditions.checkArgument(dataset instanceof BatchReadable, "Dataset '%s' is not a BatchReadable.", datasetName);
    @SuppressWarnings("unchecked") final BatchReadable<K, V> delegate = (BatchReadable<K, V>) dataset;
    return new BatchReadable<K, V>() {

        @Override
        public List<Split> getSplits() {
            try {
                try {
                    return delegate.getSplits();
                } finally {
                    flushOperations();
                }
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }

        @Override
        public SplitReader<K, V> createSplitReader(Split split) {
            return new ForwardingSplitReader<K, V>(delegate.createSplitReader(split)) {

                @Override
                public void close() {
                    try {
                        try {
                            super.close();
                        } finally {
                            flushOperations();
                        }
                    } catch (Exception e) {
                        throw Throwables.propagate(e);
                    }
                }
            };
        }
    };
}

Example 14 with Dataset

use of io.cdap.cdap.api.dataset.Dataset in project cdap by cdapio.

the class Outputs method transform.

public static ProvidedOutput transform(Output.DatasetOutput datasetOutput, AbstractContext abstractContext) {
    String datasetNamespace = datasetOutput.getNamespace();
    if (datasetNamespace == null) {
        datasetNamespace = abstractContext.getNamespace();
    }
    String datasetName = datasetOutput.getName();
    Map<String, String> args = datasetOutput.getArguments();
    Dataset dataset = abstractContext.getDataset(datasetNamespace, datasetName, args, AccessType.WRITE);
    DatasetOutputFormatProvider datasetOutputFormatProvider = new DatasetOutputFormatProvider(datasetNamespace, datasetName, args, dataset);
    return new ProvidedOutput(datasetOutput, datasetOutputFormatProvider);
}

Example 15 with Dataset

use of io.cdap.cdap.api.dataset.Dataset in project cdap by cdapio.

the class SingleThreadDatasetCache method createDatasetInstance.

/**
 * Creates a new instance of a dataset based on the given information.
 */
private Dataset createDatasetInstance(DatasetCacheKey key, boolean recordLineage) {
    DatasetId datasetId = new DatasetId(key.getNamespace(), key.getName());
    Dataset dataset = instantiator.getDataset(datasetId, key.getArguments(), key.getAccessType());
    if (dataset instanceof MeteredDataset && metricsContext != null) {
        ((MeteredDataset) dataset).setMetricsCollector(metricsContext.childContext(Constants.Metrics.Tag.DATASET, key.getName()));
    }
    if (recordLineage) {
        instantiator.writeLineage(datasetId, key.getAccessType());
    }
    return dataset;
}