Examples with HBaseProperties uk.gov.gchq.gaffer.hbasestore.HBaseProperties used on opensource projects

Example 1 with HBaseProperties

use of uk.gov.gchq.gaffer.hbasestore.HBaseProperties in project Gaffer by gchq.

the class AddElementsHandlerTest method shouldThrowNoExceptionsWhenValidateFlagSetToFalse.

@Test
public void shouldThrowNoExceptionsWhenValidateFlagSetToFalse() throws OperationException, StoreException {
    final AddElements addElements = new AddElements.Builder().input(new Edge("Unknown group", "source", "dest", true)).validate(false).build();
    final AddElementsHandler handler = new AddElementsHandler();
    final Context context = mock(Context.class);
    final HBaseStore store = mock(HBaseStore.class);
    final Table table = mock(Table.class);
    given(store.getTable()).willReturn(table);
    final HBaseProperties properties = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(getClass()));
    given(store.getProperties()).willReturn(properties);
    given(store.getSchema()).willReturn(SCHEMA);
    // When / Then - no exceptions
    handler.doOperation(addElements, context, store);
}

Example 2 with HBaseProperties

use of uk.gov.gchq.gaffer.hbasestore.HBaseProperties in project Gaffer by gchq.

the class AddElementsHandlerTest method shouldDoNothingIfNoElementsProvided.

@Test
public void shouldDoNothingIfNoElementsProvided() throws OperationException, StoreException, IOException {
    // Given
    final AddElementsHandler handler = new AddElementsHandler();
    final AddElements addElements = new AddElements();
    final Context context = mock(Context.class);
    final HBaseStore store = mock(HBaseStore.class);
    final Table table = mock(Table.class);
    given(store.getTable()).willReturn(table);
    final HBaseProperties properties = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(getClass()));
    given(store.getProperties()).willReturn(properties);
    given(store.getSchema()).willReturn(SCHEMA);
    // When
    handler.doOperation(addElements, context, store);
    // Then
    verify(table, never()).put(any(Put.class));
}

Example 3 with HBaseProperties

use of uk.gov.gchq.gaffer.hbasestore.HBaseProperties in project Gaffer by gchq.

the class HBaseAddElementsFromHdfsJobFactoryTest method shouldSetupJob.

@Test
public void shouldSetupJob() throws IOException, StoreException {
    // Given
    final JobConf localConf = createLocalConf();
    final FileSystem fs = FileSystem.getLocal(localConf);
    fs.mkdirs(new Path(outputDir));
    final JobFactory factory = getJobFactory();
    final Job job = mock(Job.class);
    final MapReduce operation = getMapReduceOperation();
    final HBaseStore store = new SingleUseMiniHBaseStore();
    final Schema schema = Schema.fromJson(StreamUtil.schemas(getClass()));
    final HBaseProperties properties = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(getClass()));
    store.initialise("graphId", schema, properties);
    given(job.getConfiguration()).willReturn(localConf);
    // When
    factory.setupJob(job, operation, TextMapperGeneratorImpl.class.getName(), store);
    // Then
    verify(job).setJarByClass(factory.getClass());
    verify(job).setJobName("Ingest HDFS data: Generator=" + TextMapperGeneratorImpl.class.getName() + ", output=" + outputDir);
    verify(job).setMapperClass(AddElementsFromHdfsMapper.class);
    verify(job).setMapOutputKeyClass(ImmutableBytesWritable.class);
    verify(job).setMapOutputValueClass(KeyValue.class);
    verify(job).setReducerClass(AddElementsFromHdfsReducer.class);
    verify(job).setOutputKeyClass(ImmutableBytesWritable.class);
    verify(job).setOutputValueClass(KeyValue.class);
    verify(job).setOutputFormatClass(HFileOutputFormat2.class);
    assertEquals(fs.makeQualified(new Path(outputDir)).toString(), job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir"));
    verify(job).setNumReduceTasks(1);
}

Example 4 with HBaseProperties

use of uk.gov.gchq.gaffer.hbasestore.HBaseProperties in project Gaffer by gchq.

the class TableUtilsTest method shouldCreateTableAndValidateIt.

@Test
public void shouldCreateTableAndValidateIt() throws Exception {
    // Given
    final SingleUseMiniHBaseStore store = new SingleUseMiniHBaseStore();
    final Schema schema = new Schema.Builder().type(TestTypes.ID_STRING, new TypeDefinition.Builder().aggregateFunction(new StringConcat()).clazz(String.class).build()).type(TestTypes.DIRECTED_TRUE, Boolean.class).edge(TestGroups.EDGE, new SchemaEdgeDefinition.Builder().source(TestTypes.ID_STRING).destination(TestTypes.ID_STRING).directed(TestTypes.DIRECTED_TRUE).build()).build();
    final HBaseProperties props = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(TableUtilsTest.class));
    store.initialise(GRAPH_ID, schema, props);
    // When
    TableUtils.createTable(store);
    TableUtils.ensureTableExists(store);
// Then - no exceptions
}

Example 5 with HBaseProperties

use of uk.gov.gchq.gaffer.hbasestore.HBaseProperties in project Gaffer by gchq.

the class TableUtils method main.

/**
 * Utility for creating and updating an HBase table.
 * See the HBase Store README for more information on what changes to your schema you are allowed to make.
 * HBase tables are automatically created when the Gaffer HBase store is initialised when an instance of Graph is created.
 * <p>
 * Running this with an existing table will remove the existing Gaffer Coprocessor and recreate it.
 * </p>
 * <p>
 * A FileGraphLibrary path must be specified as an argument.  If no path is set NoGraphLibrary will be used.
 * </p>
 * <p>
 * Usage:
 * </p>
 * <p>
 * java -cp hbase-store-[version]-utility.jar uk.gov.gchq.gaffer.hbasestore.utils.TableUtils [graphId] [pathToSchemaDirectory] [pathToStoreProperties] [pathToFileGraphLibrary]
 * </p>
 *
 * @param args [graphId] [schema directory path] [store properties path] [ file graph library path]
 * @throws Exception if the tables fails to be created/updated
 */
public static void main(final String[] args) throws Exception {
    if (args.length < NUM_REQUIRED_ARGS) {
        System.err.println("Wrong number of arguments. \nUsage: " + "<graphId> <schema directory path> <store properties path> <file graph library path>");
        System.exit(1);
    }
    final HBaseProperties storeProps = HBaseProperties.loadStoreProperties(getStorePropertiesPathString(args));
    if (null == storeProps) {
        throw new IllegalArgumentException("Store properties are required to create a store");
    }
    final Schema schema = Schema.fromJson(getSchemaDirectoryPath(args));
    GraphLibrary library;
    if (null == getFileGraphLibraryPathString(args)) {
        library = new NoGraphLibrary();
    } else {
        library = new FileGraphLibrary(getFileGraphLibraryPathString(args));
    }
    library.addOrUpdate(getGraphId(args), schema, storeProps);
    final String storeClass = storeProps.getStoreClass();
    if (null == storeClass) {
        throw new IllegalArgumentException("The Store class name was not found in the store properties for key: " + StoreProperties.STORE_CLASS);
    }
    final HBaseStore store;
    try {
        store = Class.forName(storeClass).asSubclass(HBaseStore.class).newInstance();
    } catch (final InstantiationException | IllegalAccessException | ClassNotFoundException e) {
        throw new IllegalArgumentException("Could not create store of type: " + storeClass, e);
    }
    store.preInitialise(getGraphId(args), schema, storeProps);
    if (!store.getConnection().getAdmin().tableExists(store.getTableName())) {
        createTable(store);
    }
    try (final Admin admin = store.getConnection().getAdmin()) {
        final TableName tableName = store.getTableName();
        if (admin.tableExists(tableName)) {
            final HTableDescriptor descriptor = admin.getTableDescriptor(tableName);
            descriptor.removeCoprocessor(GafferCoprocessor.class.getName());
            addCoprocesssor(descriptor, store);
            admin.modifyTable(tableName, descriptor);
        } else {
            TableUtils.createTable(store);
        }
    }
}