Examples with HBaseStore uk.gov.gchq.gaffer.hbasestore.HBaseStore used on opensource projects

Example 6 with HBaseStore

use of uk.gov.gchq.gaffer.hbasestore.HBaseStore in project Gaffer by gchq.

the class GetElementsHandlerTest method shouldReturnHBaseRetrieverWithoutIncludeMatchdxVertex.

@Test
public void shouldReturnHBaseRetrieverWithoutIncludeMatchdxVertex() throws OperationException, StoreException {
    // Given
    final Iterable<EntityId> ids = mock(Iterable.class);
    final Context context = mock(Context.class);
    final User user = mock(User.class);
    final HBaseStore store = mock(HBaseStore.class);
    final HBaseRetriever<GetElements> hbaseRetriever = mock(HBaseRetriever.class);
    final GetElementsHandler handler = new GetElementsHandler();
    final GetElements getElements = new GetElements.Builder().inputIds(ids).seedMatching(SeedMatching.SeedMatchingType.EQUAL).build();
    given(context.getUser()).willReturn(user);
    given(store.createRetriever(getElements, user, ids, false)).willReturn(hbaseRetriever);
    // When
    final HBaseRetriever<GetElements> result = (HBaseRetriever<GetElements>) handler.doOperation(getElements, context, store);
    // Then
    assertSame(hbaseRetriever, result);
}

Example 7 with HBaseStore

use of uk.gov.gchq.gaffer.hbasestore.HBaseStore in project Gaffer by gchq.

the class GetElementsHandlerTest method shouldReturnHBaseRetrieverWithIncludeMatchedVertex.

@Test
public void shouldReturnHBaseRetrieverWithIncludeMatchedVertex() throws OperationException, StoreException {
    // Given
    final Iterable<EntityId> ids = mock(Iterable.class);
    final Context context = mock(Context.class);
    final User user = mock(User.class);
    final HBaseStore store = mock(HBaseStore.class);
    final HBaseRetriever<GetElements> hbaseRetriever = mock(HBaseRetriever.class);
    final GetElementsHandler handler = new GetElementsHandler();
    final GetElements getElements = new GetElements.Builder().inputIds(ids).seedMatching(SeedMatching.SeedMatchingType.RELATED).build();
    given(context.getUser()).willReturn(user);
    given(store.createRetriever(getElements, user, ids, true)).willReturn(hbaseRetriever);
    // When
    final HBaseRetriever<GetElements> result = (HBaseRetriever<GetElements>) handler.doOperation(getElements, context, store);
    // Then
    assertSame(hbaseRetriever, result);
}

Example 8 with HBaseStore

use of uk.gov.gchq.gaffer.hbasestore.HBaseStore in project Gaffer by gchq.

the class HBaseAddElementsFromHdfsJobFactoryTest method shouldSetupJob.

@Test
public void shouldSetupJob() throws IOException, StoreException {
    // Given
    final JobConf localConf = createLocalConf();
    final FileSystem fs = FileSystem.getLocal(localConf);
    fs.mkdirs(new Path(outputDir));
    final JobFactory factory = getJobFactory();
    final Job job = mock(Job.class);
    final MapReduce operation = getMapReduceOperation();
    final HBaseStore store = new SingleUseMiniHBaseStore();
    final Schema schema = Schema.fromJson(StreamUtil.schemas(getClass()));
    final HBaseProperties properties = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(getClass()));
    store.initialise("graphId", schema, properties);
    given(job.getConfiguration()).willReturn(localConf);
    // When
    factory.setupJob(job, operation, TextMapperGeneratorImpl.class.getName(), store);
    // Then
    verify(job).setJarByClass(factory.getClass());
    verify(job).setJobName("Ingest HDFS data: Generator=" + TextMapperGeneratorImpl.class.getName() + ", output=" + outputDir);
    verify(job).setMapperClass(AddElementsFromHdfsMapper.class);
    verify(job).setMapOutputKeyClass(ImmutableBytesWritable.class);
    verify(job).setMapOutputValueClass(KeyValue.class);
    verify(job).setReducerClass(AddElementsFromHdfsReducer.class);
    verify(job).setOutputKeyClass(ImmutableBytesWritable.class);
    verify(job).setOutputValueClass(KeyValue.class);
    verify(job).setOutputFormatClass(HFileOutputFormat2.class);
    assertEquals(fs.makeQualified(new Path(outputDir)).toString(), job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir"));
    verify(job).setNumReduceTasks(1);
}

Example 9 with HBaseStore

use of uk.gov.gchq.gaffer.hbasestore.HBaseStore in project Gaffer by gchq.

the class TableUtils method main.

/**
 * Utility for creating and updating an HBase table.
 * See the HBase Store README for more information on what changes to your schema you are allowed to make.
 * HBase tables are automatically created when the Gaffer HBase store is initialised when an instance of Graph is created.
 * <p>
 * Running this with an existing table will remove the existing Gaffer Coprocessor and recreate it.
 * </p>
 * <p>
 * A FileGraphLibrary path must be specified as an argument.  If no path is set NoGraphLibrary will be used.
 * </p>
 * <p>
 * Usage:
 * </p>
 * <p>
 * java -cp hbase-store-[version]-utility.jar uk.gov.gchq.gaffer.hbasestore.utils.TableUtils [graphId] [pathToSchemaDirectory] [pathToStoreProperties] [pathToFileGraphLibrary]
 * </p>
 *
 * @param args [graphId] [schema directory path] [store properties path] [ file graph library path]
 * @throws Exception if the tables fails to be created/updated
 */
public static void main(final String[] args) throws Exception {
    if (args.length < NUM_REQUIRED_ARGS) {
        System.err.println("Wrong number of arguments. \nUsage: " + "<graphId> <schema directory path> <store properties path> <file graph library path>");
        System.exit(1);
    }
    final HBaseProperties storeProps = HBaseProperties.loadStoreProperties(getStorePropertiesPathString(args));
    if (null == storeProps) {
        throw new IllegalArgumentException("Store properties are required to create a store");
    }
    final Schema schema = Schema.fromJson(getSchemaDirectoryPath(args));
    GraphLibrary library;
    if (null == getFileGraphLibraryPathString(args)) {
        library = new NoGraphLibrary();
    } else {
        library = new FileGraphLibrary(getFileGraphLibraryPathString(args));
    }
    library.addOrUpdate(getGraphId(args), schema, storeProps);
    final String storeClass = storeProps.getStoreClass();
    if (null == storeClass) {
        throw new IllegalArgumentException("The Store class name was not found in the store properties for key: " + StoreProperties.STORE_CLASS);
    }
    final HBaseStore store;
    try {
        store = Class.forName(storeClass).asSubclass(HBaseStore.class).newInstance();
    } catch (final InstantiationException | IllegalAccessException | ClassNotFoundException e) {
        throw new IllegalArgumentException("Could not create store of type: " + storeClass, e);
    }
    store.preInitialise(getGraphId(args), schema, storeProps);
    if (!store.getConnection().getAdmin().tableExists(store.getTableName())) {
        createTable(store);
    }
    try (final Admin admin = store.getConnection().getAdmin()) {
        final TableName tableName = store.getTableName();
        if (admin.tableExists(tableName)) {
            final HTableDescriptor descriptor = admin.getTableDescriptor(tableName);
            descriptor.removeCoprocessor(GafferCoprocessor.class.getName());
            addCoprocesssor(descriptor, store);
            admin.modifyTable(tableName, descriptor);
        } else {
            TableUtils.createTable(store);
        }
    }
}

Example 10 with HBaseStore

use of uk.gov.gchq.gaffer.hbasestore.HBaseStore in project Gaffer by gchq.

the class AddElementsHandlerTest method shouldAddElements.

@Test
public void shouldAddElements() throws OperationException, StoreException, IOException {
    // Given
    final AddElementsHandler handler = new AddElementsHandler();
    final List<Element> elements = createElements();
    final List<Element> elementsWithNull = new ArrayList<>(elements);
    // null should be skipped
    elementsWithNull.add(null);
    final AddElements addElements = new AddElements.Builder().input(elementsWithNull).build();
    final Context context = mock(Context.class);
    final HBaseStore store = mock(HBaseStore.class);
    final HTable table = mock(HTable.class);
    given(store.getTable()).willReturn(table);
    final HBaseProperties properties = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(getClass()));
    final int writeBufferSize = 5;
    properties.setWriteBufferSize(writeBufferSize);
    given(store.getProperties()).willReturn(properties);
    given(store.getSchema()).willReturn(SCHEMA);
    // When
    handler.doOperation(addElements, context, store);
    // Then
    final ArgumentCaptor<List<Put>> putsCaptor = (ArgumentCaptor) ArgumentCaptor.forClass(List.class);
    verify(table, times(2)).put(putsCaptor.capture());
    verify(table, times(2)).flushCommits();
    final List<List<Put>> allPuts = putsCaptor.getAllValues();
    assertThat(allPuts).hasSize(2);
    final List<Put> combinedPuts = new ArrayList<>();
    combinedPuts.addAll(allPuts.get(0));
    combinedPuts.addAll(allPuts.get(1));
    final List<Element> expectedElements = new ArrayList<>();
    for (final Element element : elements) {
        expectedElements.add(element);
        if (element instanceof Edge && !((Edge) element).getSource().equals(((Edge) element).getDestination())) {
            expectedElements.add(element);
        }
    }
    final Element[] expectedElementsArr = expectedElements.toArray(new Element[expectedElements.size()]);
    final List<Element> elementsAdded = CellUtil.getElements(combinedPuts, new ElementSerialisation(SCHEMA), false);
    assertEquals(expectedElements.size(), elementsAdded.size());
    assertThat(elementsAdded).contains(expectedElementsArr);
}