Examples with FileDataIngest org.apache.accumulo.examples.filedata.FileDataIngest used on opensource projects

Example 1 with FileDataIngest

use of org.apache.accumulo.examples.filedata.FileDataIngest in project accumulo-examples by apache.

the class Ingest method main.

public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(Ingest.class.getName(), args, bwOpts);
    Connector conn = opts.getConnector();
    if (!conn.tableOperations().exists(opts.nameTable))
        conn.tableOperations().create(opts.nameTable);
    if (!conn.tableOperations().exists(opts.indexTable))
        conn.tableOperations().create(opts.indexTable);
    if (!conn.tableOperations().exists(opts.dataTable)) {
        conn.tableOperations().create(opts.dataTable);
        conn.tableOperations().attachIterator(opts.dataTable, new IteratorSetting(1, ChunkCombiner.class));
    }
    BatchWriter dirBW = conn.createBatchWriter(opts.nameTable, bwOpts.getBatchWriterConfig());
    BatchWriter indexBW = conn.createBatchWriter(opts.indexTable, bwOpts.getBatchWriterConfig());
    BatchWriter dataBW = conn.createBatchWriter(opts.dataTable, bwOpts.getBatchWriterConfig());
    FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
    for (String dir : opts.directories) {
        recurse(new File(dir), opts.visibility, dirBW, indexBW, fdi, dataBW);
        // fill in parent directory info
        int slashIndex = -1;
        while ((slashIndex = dir.lastIndexOf("/")) > 0) {
            dir = dir.substring(0, slashIndex);
            ingest(new File(dir), opts.visibility, dirBW, indexBW, fdi, dataBW);
        }
    }
    ingest(new File("/"), opts.visibility, dirBW, indexBW, fdi, dataBW);
    dirBW.close();
    indexBW.close();
    dataBW.close();
}