Examples with SingleUseMiniHBaseStore uk.gov.gchq.gaffer.hbasestore.SingleUseMiniHBaseStore used on opensource projects

Example 1 with SingleUseMiniHBaseStore

use of uk.gov.gchq.gaffer.hbasestore.SingleUseMiniHBaseStore in project Gaffer by gchq.

the class HBaseAddElementsFromHdfsJobFactoryTest method shouldSetupJob.

@Test
public void shouldSetupJob() throws IOException, StoreException {
    // Given
    final JobConf localConf = createLocalConf();
    final FileSystem fs = FileSystem.getLocal(localConf);
    fs.mkdirs(new Path(outputDir));
    final JobFactory factory = getJobFactory();
    final Job job = mock(Job.class);
    final MapReduce operation = getMapReduceOperation();
    final HBaseStore store = new SingleUseMiniHBaseStore();
    final Schema schema = Schema.fromJson(StreamUtil.schemas(getClass()));
    final HBaseProperties properties = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(getClass()));
    store.initialise("graphId", schema, properties);
    given(job.getConfiguration()).willReturn(localConf);
    // When
    factory.setupJob(job, operation, TextMapperGeneratorImpl.class.getName(), store);
    // Then
    verify(job).setJarByClass(factory.getClass());
    verify(job).setJobName("Ingest HDFS data: Generator=" + TextMapperGeneratorImpl.class.getName() + ", output=" + outputDir);
    verify(job).setMapperClass(AddElementsFromHdfsMapper.class);
    verify(job).setMapOutputKeyClass(ImmutableBytesWritable.class);
    verify(job).setMapOutputValueClass(KeyValue.class);
    verify(job).setReducerClass(AddElementsFromHdfsReducer.class);
    verify(job).setOutputKeyClass(ImmutableBytesWritable.class);
    verify(job).setOutputValueClass(KeyValue.class);
    verify(job).setOutputFormatClass(HFileOutputFormat2.class);
    assertEquals(fs.makeQualified(new Path(outputDir)).toString(), job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir"));
    verify(job).setNumReduceTasks(1);
}

Example 2 with SingleUseMiniHBaseStore

use of uk.gov.gchq.gaffer.hbasestore.SingleUseMiniHBaseStore in project Gaffer by gchq.

the class TableUtilsTest method shouldCreateTableAndValidateIt.

@Test
public void shouldCreateTableAndValidateIt() throws Exception {
    // Given
    final SingleUseMiniHBaseStore store = new SingleUseMiniHBaseStore();
    final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, new TypeDefinition.Builder().aggregateFunction(new StringConcat()).clazz(String.class).build()).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
    final HBaseProperties props = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
    store.initialise(GRAPH_ID, schema, props);
    // When
    TableUtils.createTable(store);
    TableUtils.ensureTableExists(store);
// Then - no exceptions
}

Example 3 with SingleUseMiniHBaseStore

use of uk.gov.gchq.gaffer.hbasestore.SingleUseMiniHBaseStore in project Gaffer by gchq.

the class HBaseAddElementsFromHdfsJobFactoryTest method getStoreConfiguredWith.

@Override
protected Store getStoreConfiguredWith(final Class<JSONSerialiser> jsonSerialiserClass, final String jsonSerialiserModules, final Boolean strictJson) throws IOException, StoreException {
    final HBaseStore store = new SingleUseMiniHBaseStore();
    final Schema schema = Schema.fromJson(StreamUtil.schemas(getClass()));
    final HBaseProperties properties = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(getClass()));
    super.configureStoreProperties(properties, jsonSerialiserClass, jsonSerialiserModules, strictJson);
    store.initialise("graphId", schema, properties);
    final JobConf localConf = createLocalConf();
    final FileSystem fs = FileSystem.getLocal(localConf);
    fs.mkdirs(new Path(outputDir));
    return store;
}

Example 4 with SingleUseMiniHBaseStore

use of uk.gov.gchq.gaffer.hbasestore.SingleUseMiniHBaseStore in project Gaffer by gchq.

the class TableUtilsTest method shouldFailTableValidationWhenTableDoesntHaveCoprocessor.

@Test
public void shouldFailTableValidationWhenTableDoesntHaveCoprocessor() throws Exception {
    // Given
    final SingleUseMiniHBaseStore store = new SingleUseMiniHBaseStore();
    final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, new TypeDefinition.Builder().aggregateFunction(new StringConcat()).clazz(String.class).build()).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
    final HBaseProperties props = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
    store.initialise(GRAPH_ID, schema, props);
    // Remove coprocessor
    final TableName tableName = store.getTableName();
    try (final Admin admin = store.getConnection().getAdmin()) {
        final HTableDescriptor descriptor = admin.getTableDescriptor(tableName);
        descriptor.removeCoprocessor(GafferCoprocessor.class.getName());
        admin.modifyTable(tableName, descriptor);
    } catch (final StoreException | IOException e) {
        throw new RuntimeException(e);
    }
    // When / Then
    assertThatExceptionOfType(StoreException.class).isThrownBy(() -> TableUtils.ensureTableExists(store)).extracting("message").isNotNull();
}