use of com.thinkaurelius.titan.hadoop.scan.CassandraHadoopScanRunner in project titan by thinkaurelius.
the class CassandraScanJobIT method testSimpleScan.
@Test
public void testSimpleScan() throws InterruptedException, ExecutionException, IOException, BackendException {
int keys = 1000;
int cols = 40;
String[][] values = KeyValueStoreUtil.generateData(keys, cols);
//Make it only half the number of columns for every 2nd key
for (int i = 0; i < values.length; i++) {
if (i % 2 == 0)
values[i] = Arrays.copyOf(values[i], cols / 2);
}
log.debug("Loading values: " + keys + "x" + cols);
KeyColumnValueStoreManager mgr = new CassandraThriftStoreManager(GraphDatabaseConfiguration.buildGraphConfiguration());
KeyColumnValueStore store = mgr.openDatabase("edgestore");
StoreTransaction tx = mgr.beginTransaction(StandardBaseTransactionConfig.of(TimestampProviders.MICRO));
KeyColumnValueStoreUtil.loadValues(store, tx, values);
// noop on Cassandra, but harmless
tx.commit();
SimpleScanJobRunner runner = (ScanJob job, Configuration jobConf, String rootNSName) -> {
try {
return new CassandraHadoopScanRunner(job).scanJobConf(jobConf).scanJobConfRoot(rootNSName).partitionerOverride("org.apache.cassandra.dht.Murmur3Partitioner").run();
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
};
SimpleScanJob.runBasicTests(keys, cols, runner);
}
use of com.thinkaurelius.titan.hadoop.scan.CassandraHadoopScanRunner in project titan by thinkaurelius.
the class MapReduceIndexJobs method cassandraRepair.
public static ScanMetrics cassandraRepair(Properties titanProperties, String indexName, String relationType, String partitionerName, Configuration hadoopBaseConf) throws InterruptedException, IOException, ClassNotFoundException {
IndexRepairJob job = new IndexRepairJob();
CassandraHadoopScanRunner cr = new CassandraHadoopScanRunner(job);
ModifiableConfiguration mc = getIndexJobConf(indexName, relationType);
copyPropertiesToInputAndOutputConf(hadoopBaseConf, titanProperties);
cr.partitionerOverride(partitionerName);
cr.scanJobConf(mc);
cr.scanJobConfRoot(GraphDatabaseConfiguration.class.getName() + "#JOB_NS");
cr.baseHadoopConf(hadoopBaseConf);
return cr.run();
}
use of com.thinkaurelius.titan.hadoop.scan.CassandraHadoopScanRunner in project titan by thinkaurelius.
the class MapReduceIndexJobs method cassandraRemove.
public static ScanMetrics cassandraRemove(Properties titanProperties, String indexName, String relationType, String partitionerName, Configuration hadoopBaseConf) throws InterruptedException, IOException, ClassNotFoundException {
IndexRemoveJob job = new IndexRemoveJob();
CassandraHadoopScanRunner cr = new CassandraHadoopScanRunner(job);
ModifiableConfiguration mc = getIndexJobConf(indexName, relationType);
copyPropertiesToInputAndOutputConf(hadoopBaseConf, titanProperties);
cr.partitionerOverride(partitionerName);
cr.scanJobConf(mc);
cr.scanJobConfRoot(GraphDatabaseConfiguration.class.getName() + "#JOB_NS");
cr.baseHadoopConf(hadoopBaseConf);
return cr.run();
}
Aggregations