Examples with RFileWriter org.apache.accumulo.core.client.rfile.RFileWriter used on opensource projects

Example 1 with RFileWriter

use of org.apache.accumulo.core.client.rfile.RFileWriter in project accumulo by apache.

the class AccumuloFileOutputFormat method getRecordWriter.

@Override
public RecordWriter<Key, Value> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
    // get the path of the temporary output file
    final Configuration conf = job;
    final AccumuloConfiguration acuConf = FileOutputConfigurator.getAccumuloConfiguration(CLASS, job);
    final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
    final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension);
    final int visCacheSize = FileOutputConfigurator.getVisibilityCacheSize(conf);
    return new RecordWriter<>() {

        RFileWriter out = null;

        @Override
        public void close(Reporter reporter) throws IOException {
            if (out != null)
                out.close();
        }

        @Override
        public void write(Key key, Value value) throws IOException {
            if (out == null) {
                out = RFile.newWriter().to(file.toString()).withFileSystem(file.getFileSystem(conf)).withTableProperties(acuConf).withVisibilityCacheSize(visCacheSize).build();
                out.startDefaultLocalityGroup();
            }
            out.append(key, value);
        }
    };
}

Example 2 with RFileWriter

use of org.apache.accumulo.core.client.rfile.RFileWriter in project accumulo by apache.

the class CryptoTest method testReadNoCryptoWithCryptoConfigured.

@Test
public // This test is to ensure when Crypto is configured that it can read unencrypted files
void testReadNoCryptoWithCryptoConfigured() throws Exception {
    AccumuloConfiguration cryptoOffConf = getAccumuloConfig(ConfigMode.CRYPTO_OFF);
    AccumuloConfiguration cryptoOnConf = getAccumuloConfig(ConfigMode.CRYPTO_ON);
    FileSystem fs = FileSystem.getLocal(hadoopConf);
    ArrayList<Key> keys = testData();
    String file = "target/testFile2.rf";
    fs.delete(new Path(file), true);
    try (RFileWriter writer = RFile.newWriter().to(file).withFileSystem(fs).withTableProperties(cryptoOffConf).build()) {
        Value empty = new Value();
        writer.startDefaultLocalityGroup();
        for (Key key : keys) {
            writer.append(key, empty);
        }
    }
    Scanner iter = RFile.newScanner().from(file).withFileSystem(fs).withTableProperties(cryptoOnConf).build();
    ArrayList<Key> keysRead = new ArrayList<>();
    iter.forEach(e -> keysRead.add(e.getKey()));
    assertEquals(keys, keysRead);
}

Example 3 with RFileWriter

use of org.apache.accumulo.core.client.rfile.RFileWriter in project accumulo by apache.

the class AccumuloFileOutputFormat method getRecordWriter.

@Override
public RecordWriter<Key, Value> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) {
    // get the path of the temporary output file
    final Configuration conf = job;
    final AccumuloConfiguration acuConf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job);
    final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
    final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension);
    final int visCacheSize = ConfiguratorBase.getVisibilityCacheSize(conf);
    return new RecordWriter<>() {

        RFileWriter out = null;

        @Override
        public void close(Reporter reporter) throws IOException {
            if (out != null)
                out.close();
        }

        @Override
        public void write(Key key, Value value) throws IOException {
            if (out == null) {
                out = RFile.newWriter().to(file.toString()).withFileSystem(file.getFileSystem(conf)).withTableProperties(acuConf).withVisibilityCacheSize(visCacheSize).build();
                out.startDefaultLocalityGroup();
            }
            out.append(key, value);
        }
    };
}

Example 4 with RFileWriter

use of org.apache.accumulo.core.client.rfile.RFileWriter in project accumulo by apache.

the class AccumuloFileOutputFormat method getRecordWriter.

@Override
public RecordWriter<Key, Value> getRecordWriter(TaskAttemptContext context) throws IOException {
    // get the path of the temporary output file
    final Configuration conf = context.getConfiguration();
    final AccumuloConfiguration acuConf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, context.getConfiguration());
    final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
    final Path file = this.getDefaultWorkFile(context, "." + extension);
    final int visCacheSize = ConfiguratorBase.getVisibilityCacheSize(conf);
    return new RecordWriter<>() {

        RFileWriter out = null;

        @Override
        public void close(TaskAttemptContext context) throws IOException {
            if (out != null)
                out.close();
        }

        @Override
        public void write(Key key, Value value) throws IOException {
            if (out == null) {
                out = RFile.newWriter().to(file.toString()).withFileSystem(file.getFileSystem(conf)).withTableProperties(acuConf).withVisibilityCacheSize(visCacheSize).build();
                out.startDefaultLocalityGroup();
            }
            out.append(key, value);
        }
    };
}

Example 5 with RFileWriter

use of org.apache.accumulo.core.client.rfile.RFileWriter in project accumulo by apache.

the class AccumuloFileOutputFormat method getRecordWriter.

@Override
public RecordWriter<Key, Value> getRecordWriter(TaskAttemptContext context) throws IOException {
    // get the path of the temporary output file
    final Configuration conf = context.getConfiguration();
    final AccumuloConfiguration acuConf = FileOutputConfigurator.getAccumuloConfiguration(CLASS, context.getConfiguration());
    final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
    final Path file = this.getDefaultWorkFile(context, "." + extension);
    final int visCacheSize = FileOutputConfigurator.getVisibilityCacheSize(conf);
    return new RecordWriter<>() {

        RFileWriter out = null;

        @Override
        public void close(TaskAttemptContext context) throws IOException {
            if (out != null)
                out.close();
        }

        @Override
        public void write(Key key, Value value) throws IOException {
            if (out == null) {
                out = RFile.newWriter().to(file.toString()).withFileSystem(file.getFileSystem(conf)).withTableProperties(acuConf).withVisibilityCacheSize(visCacheSize).build();
                out.startDefaultLocalityGroup();
            }
            out.append(key, value);
        }
    };
}