Examples with CryptoService org.apache.accumulo.core.spi.crypto.CryptoService used on opensource projects

Example 6 with CryptoService

use of org.apache.accumulo.core.spi.crypto.CryptoService in project accumulo by apache.

the class FileSystemInitializer method createMetadataFile.

private void createMetadataFile(VolumeManager volmanager, String fileName, AccumuloConfiguration conf, Tablet... tablets) throws IOException {
    // sort file contents in memory, then play back to the file
    TreeMap<Key, Value> sorted = new TreeMap<>();
    for (Tablet tablet : tablets) {
        createEntriesForTablet(sorted, tablet);
    }
    FileSystem fs = volmanager.getFileSystemByPath(new Path(fileName));
    CryptoService cs = CryptoServiceFactory.newInstance(conf, CryptoServiceFactory.ClassloaderType.ACCUMULO);
    FileSKVWriter tabletWriter = FileOperations.getInstance().newWriterBuilder().forFile(fileName, fs, fs.getConf(), cs).withTableConfiguration(conf).build();
    tabletWriter.startDefaultLocalityGroup();
    for (Map.Entry<Key, Value> entry : sorted.entrySet()) {
        tabletWriter.append(entry.getKey(), entry.getValue());
    }
    tabletWriter.close();
}

Example 7 with CryptoService

use of org.apache.accumulo.core.spi.crypto.CryptoService in project accumulo by apache.

the class DfsLogger method open.

/**
 * Opens a Write-Ahead Log file and writes the necessary header information and OPEN entry to the
 * file. The file is ready to be used for ingest if this method returns successfully. If an
 * exception is thrown from this method, it is the callers responsibility to ensure that
 * {@link #close()} is called to prevent leaking the file handle and/or syncing thread.
 *
 * @param address
 *          The address of the host using this WAL
 */
public synchronized void open(String address) throws IOException {
    String filename = UUID.randomUUID().toString();
    log.debug("Address is {}", address);
    String logger = Joiner.on("+").join(address.split(":"));
    log.debug("DfsLogger.open() begin");
    VolumeManager fs = conf.getVolumeManager();
    var chooserEnv = new VolumeChooserEnvironmentImpl(org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment.Scope.LOGGER, context);
    logPath = fs.choose(chooserEnv, context.getBaseUris()) + Path.SEPARATOR + Constants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
    metaReference = toString();
    LoggerOperation op = null;
    try {
        Path logfilePath = new Path(logPath);
        short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
        if (replication == 0)
            replication = fs.getDefaultReplication(logfilePath);
        long blockSize = getWalBlockSize(conf.getConfiguration());
        if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
            logFile = fs.createSyncable(logfilePath, 0, replication, blockSize);
        else
            logFile = fs.create(logfilePath, true, 0, replication, blockSize);
        // check again that logfile can be sync'd
        if (!fs.canSyncAndFlush(logfilePath)) {
            log.warn("sync not supported for log file {}. Data loss may occur.", logPath);
        }
        // Initialize the log file with a header and its encryption
        CryptoService cryptoService = context.getCryptoService();
        logFile.write(LOG_FILE_HEADER_V4.getBytes(UTF_8));
        log.debug("Using {} for encrypting WAL {}", cryptoService.getClass().getSimpleName(), filename);
        CryptoEnvironment env = new CryptoEnvironmentImpl(Scope.WAL, null);
        FileEncrypter encrypter = cryptoService.getFileEncrypter(env);
        byte[] cryptoParams = encrypter.getDecryptionParameters();
        CryptoUtils.writeParams(cryptoParams, logFile);
        /**
         * Always wrap the WAL in a NoFlushOutputStream to prevent extra flushing to HDFS. The
         * {@link #write(LogFileKey, LogFileValue)} method will flush crypto data or do nothing when
         * crypto is not enabled.
         */
        OutputStream encryptedStream = encrypter.encryptStream(new NoFlushOutputStream(logFile));
        if (encryptedStream instanceof NoFlushOutputStream) {
            encryptingLogFile = (NoFlushOutputStream) encryptedStream;
        } else {
            encryptingLogFile = new DataOutputStream(encryptedStream);
        }
        LogFileKey key = new LogFileKey();
        key.event = OPEN;
        key.tserverSession = filename;
        key.filename = filename;
        op = logKeyData(key, Durability.SYNC);
    } catch (Exception ex) {
        if (logFile != null)
            logFile.close();
        logFile = null;
        encryptingLogFile = null;
        throw new IOException(ex);
    }
    syncThread = Threads.createThread("Accumulo WALog thread " + this, new LogSyncingTask());
    syncThread.start();
    op.await();
    log.debug("Got new write-ahead log: {}", this);
}

Example 8 with CryptoService

use of org.apache.accumulo.core.spi.crypto.CryptoService in project accumulo by apache.

the class RFileSummariesRetriever method read.

@Override
public Collection<Summary> read() throws IOException {
    SummarizerFactory factory = new SummarizerFactory();
    ConfigurationCopy acuconf = new ConfigurationCopy(DefaultConfiguration.getInstance());
    config.forEach(acuconf::set);
    RFileSource[] sources = in.getSources();
    try {
        SummaryCollection all = new SummaryCollection();
        CryptoService cservice = CryptoServiceFactory.newInstance(acuconf, ClassloaderType.JAVA);
        for (int i = 0; i < sources.length; i++) {
            SummaryReader fileSummary = SummaryReader.load(in.getFileSystem().getConf(), sources[i], "source-" + i, summarySelector, factory, cservice);
            SummaryCollection sc = fileSummary.getSummaries(Collections.singletonList(new Gatherer.RowRange(startRow, endRow)));
            all.merge(sc, factory);
        }
        return all.getSummaries();
    } finally {
        for (RFileSource source : sources) {
            source.getInputStream().close();
        }
    }
}

Example 9 with CryptoService

use of org.apache.accumulo.core.spi.crypto.CryptoService in project accumulo by apache.

the class RFileWriterBuilder method build.

@Override
public RFileWriter build() throws IOException {
    FileOperations fileops = FileOperations.getInstance();
    AccumuloConfiguration acuconf = DefaultConfiguration.getInstance();
    HashMap<String, String> userProps = new HashMap<>();
    userProps.putAll(tableConfig);
    userProps.putAll(summarizerProps);
    userProps.putAll(samplerProps);
    if (!userProps.isEmpty()) {
        acuconf = new ConfigurationCopy(Iterables.concat(acuconf, userProps.entrySet()));
    }
    CryptoService cs = CryptoServiceFactory.newInstance(acuconf, ClassloaderType.JAVA);
    if (out.getOutputStream() != null) {
        FSDataOutputStream fsdo;
        if (out.getOutputStream() instanceof FSDataOutputStream) {
            fsdo = (FSDataOutputStream) out.getOutputStream();
        } else {
            fsdo = new FSDataOutputStream(out.getOutputStream(), new FileSystem.Statistics("foo"));
        }
        return new RFileWriter(fileops.newWriterBuilder().forOutputStream(".rf", fsdo, out.getConf(), cs).withTableConfiguration(acuconf).withStartDisabled().build(), visCacheSize);
    } else {
        return new RFileWriter(fileops.newWriterBuilder().forFile(out.path.toString(), out.getFileSystem(), out.getConf(), cs).withTableConfiguration(acuconf).withStartDisabled().build(), visCacheSize);
    }
}

Example 10 with CryptoService

use of org.apache.accumulo.core.spi.crypto.CryptoService in project accumulo by apache.

the class CryptoServiceFactory method newInstance.

public static CryptoService newInstance(AccumuloConfiguration conf, ClassloaderType ct) {
    CryptoService newCryptoService;
    if (ct == ClassloaderType.ACCUMULO) {
        newCryptoService = Property.createInstanceFromPropertyName(conf, Property.INSTANCE_CRYPTO_SERVICE, CryptoService.class, new NoCryptoService());
    } else if (ct == ClassloaderType.JAVA) {
        String clazzName = conf.get(Property.INSTANCE_CRYPTO_SERVICE);
        if (clazzName == null || clazzName.trim().isEmpty()) {
            newCryptoService = new NoCryptoService();
        } else {
            try {
                newCryptoService = CryptoServiceFactory.class.getClassLoader().loadClass(clazzName).asSubclass(CryptoService.class).getDeclaredConstructor().newInstance();
            } catch (ReflectiveOperationException e) {
                throw new RuntimeException(e);
            }
        }
    } else {
        throw new IllegalArgumentException();
    }
    newCryptoService.init(conf.getAllPropertiesWithPrefix(Property.INSTANCE_CRYPTO_PREFIX));
    return newCryptoService;
}