Examples with AddElementsFromHdfsTool uk.gov.gchq.gaffer.hdfs.operation.handler.job.tool.AddElementsFromHdfsTool used on opensource projects

Example 1 with AddElementsFromHdfsTool

use of uk.gov.gchq.gaffer.hdfs.operation.handler.job.tool.AddElementsFromHdfsTool in project Gaffer by gchq.

the class AddElementsFromHdfsHandler method fetchElements.

private void fetchElements(final AddElementsFromHdfs operation, final AccumuloStore store) throws OperationException {
    final int response;
    try {
        /* Parse any Hadoop arguments passed on the command line and use these to configure the Tool */
        final Configuration configuration = new GenericOptionsParser(operation.getCommandLineArgs()).getConfiguration();
        final AddElementsFromHdfsTool fetchTool = new AddElementsFromHdfsTool(new AccumuloAddElementsFromHdfsJobFactory(configuration), operation, store);
        LOGGER.info("Running FetchElementsFromHdfsTool job");
        response = ToolRunner.run(fetchTool, operation.getCommandLineArgs());
        LOGGER.info("Finished running FetchElementsFromHdfsTool job");
    } catch (final Exception e) {
        LOGGER.error("Failed to fetch elements from HDFS: {}", e.getMessage());
        throw new OperationException("Failed to fetch elements from HDFS", e);
    }
    if (AddElementsFromHdfsTool.SUCCESS_RESPONSE != response) {
        LOGGER.error("Failed to fetch elements from HDFS. Response code was {}", response);
        throw new OperationException("Failed to fetch elements from HDFS. Response code was: " + response);
    }
}

Example 2 with AddElementsFromHdfsTool

use of uk.gov.gchq.gaffer.hdfs.operation.handler.job.tool.AddElementsFromHdfsTool in project Gaffer by gchq.

the class AddElementsFromHdfsHandler method checkHdfsDirectories.

private void checkHdfsDirectories(final AddElementsFromHdfs operation, final AccumuloStore store) throws IOException {
    final AddElementsFromHdfsTool tool = new AddElementsFromHdfsTool(new AccumuloAddElementsFromHdfsJobFactory(), operation, store);
    LOGGER.info("Checking that the correct HDFS directories exist");
    final FileSystem fs = FileSystem.get(tool.getConfig());
    final Path outputPath = new Path(operation.getOutputPath());
    LOGGER.info("Ensuring output directory {} doesn't exist", outputPath);
    if (fs.exists(outputPath)) {
        if (fs.listFiles(outputPath, true).hasNext()) {
            LOGGER.error("Output directory exists and is not empty: {}", outputPath);
            throw new IllegalArgumentException("Output directory exists and is not empty: " + outputPath);
        }
        LOGGER.info("Output directory exists and is empty so deleting: {}", outputPath);
        fs.delete(outputPath, true);
    }
}

Example 3 with AddElementsFromHdfsTool

use of uk.gov.gchq.gaffer.hdfs.operation.handler.job.tool.AddElementsFromHdfsTool in project Gaffer by gchq.

the class AddElementsFromHdfsHandler method fetchElements.

private void fetchElements(final AddElementsFromHdfs operation, final HBaseStore store) throws OperationException {
    try {
        /* Parse any Hadoop arguments passed on the command line and use these to configure the Tool */
        final Configuration configuration = new GenericOptionsParser(store.getConfiguration(), operation.getCommandLineArgs()).getConfiguration();
        final AddElementsFromHdfsTool fetchTool = new AddElementsFromHdfsTool(new HBaseAddElementsFromHdfsJobFactory(configuration), operation, store);
        LOGGER.info("Running FetchElementsFromHdfsTool job");
        ToolRunner.run(fetchTool, operation.getCommandLineArgs());
        LOGGER.info("Finished running FetchElementsFromHdfsTool job");
    } catch (final Exception e) {
        LOGGER.error("Failed to fetch elements from HDFS: {}", e.getMessage());
        throw new OperationException("Failed to fetch elements from HDFS", e);
    }
}

Example 4 with AddElementsFromHdfsTool

use of uk.gov.gchq.gaffer.hdfs.operation.handler.job.tool.AddElementsFromHdfsTool in project Gaffer by gchq.

the class AddElementsFromHdfsHandler method checkHdfsDirectories.

private void checkHdfsDirectories(final AddElementsFromHdfs operation, final HBaseStore store) throws IOException {
    final AddElementsFromHdfsTool tool = new AddElementsFromHdfsTool(new HBaseAddElementsFromHdfsJobFactory(), operation, store);
    LOGGER.info("Checking that the correct HDFS directories exist");
    final FileSystem fs = FileSystem.get(tool.getConfig());
    final Path outputPath = new Path(operation.getOutputPath());
    LOGGER.info("Ensuring output directory {} doesn't exist", outputPath);
    if (fs.exists(outputPath)) {
        if (fs.listFiles(outputPath, true).hasNext()) {
            LOGGER.error("Output directory exists and is not empty: {}", outputPath);
            throw new IllegalArgumentException("Output directory exists and is not empty: " + outputPath);
        }
        LOGGER.info("Output directory exists and is empty so deleting: {}", outputPath);
        fs.delete(outputPath, true);
    }
}