Examples with DigitalSignature org.codice.ddf.commands.util.DigitalSignature used on opensource projects

Example 1 with DigitalSignature

use of org.codice.ddf.commands.util.DigitalSignature in project ddf by codice.

the class IngestCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    if (this.verifier == null) {
        this.verifier = new DigitalSignature(security);
    }
    if (batchSize * multithreaded > MAX_QUEUE_SIZE) {
        throw new IngestException(String.format("batchsize * multithreaded cannot be larger than %d.", MAX_QUEUE_SIZE));
    }
    final File inputFile = getInputFile();
    if (inputFile == null) {
        return null;
    }
    int totalFiles = totalFileCount(inputFile);
    fileCount.set(totalFiles);
    final ArrayBlockingQueue<Metacard> metacardQueue = new ArrayBlockingQueue<>(batchSize * multithreaded);
    ExecutorService queueExecutor = Executors.newSingleThreadExecutor(StandardThreadFactoryBuilder.newThreadFactory(THREAD_NAME));
    final long start = System.currentTimeMillis();
    printProgressAndFlush(start, fileCount.get(), 0);
    // Registering for the main thread and on behalf of the buildQueue thread
    // the buildQueue thread will unregister itself when the files have all
    // been added to the blocking queue and the final registration will
    // be held for the await.
    phaser.register();
    phaser.register();
    queueExecutor.submit(() -> buildQueue(inputFile, metacardQueue, start));
    final ScheduledExecutorService batchScheduler = Executors.newSingleThreadScheduledExecutor(StandardThreadFactoryBuilder.newThreadFactory(THREAD_NAME));
    BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded);
    RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
    ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, StandardThreadFactoryBuilder.newThreadFactory(THREAD_NAME), rejectedExecutionHandler);
    final CatalogFacade catalog = getCatalog();
    submitToCatalog(batchScheduler, executorService, metacardQueue, catalog, start);
    // await on catalog processing threads to complete emptying queue
    phaser.awaitAdvance(phaser.arrive());
    try {
        queueExecutor.shutdown();
        executorService.shutdown();
        batchScheduler.shutdown();
    } catch (SecurityException e) {
        LOGGER.info("Executor service shutdown was not permitted", e);
    }
    printProgressAndFlush(start, fileCount.get(), (long) ingestCount.get() + ignoreCount.get());
    long end = System.currentTimeMillis();
    console.println();
    String elapsedTime = timeFormatter.print(new Period(start, end).withMillis(0));
    console.println();
    console.printf(" %d file(s) ingested in %s %n", ingestCount.get(), elapsedTime);
    LOGGER.debug("{} file(s) ingested in {} [{} records/sec]", ingestCount.get(), elapsedTime, calculateRecordsPerSecond(ingestCount.get(), start, end));
    INGEST_LOGGER.info("{} file(s) ingested in {} [{} records/sec]", ingestCount.get(), elapsedTime, calculateRecordsPerSecond(ingestCount.get(), start, end));
    if (fileCount.get() != ingestCount.get()) {
        if ((fileCount.get() - ingestCount.get() - ignoreCount.get()) >= 1) {
            String failedAmount = Integer.toString(fileCount.get() - ingestCount.get() - ignoreCount.get());
            console.println();
            printErrorMessage(failedAmount + " file(s) failed to be ingested. See the ingest log for more details.");
            INGEST_LOGGER.warn("{} file(s) failed to be ingested.", failedAmount);
        }
        if (ignoreList != null) {
            String ignoredAmount = Integer.toString(ignoreCount.get());
            console.println();
            printColor(Ansi.Color.YELLOW, ignoredAmount + " file(s) ignored. See the ingest log for more details.");
            INGEST_LOGGER.warn("{} file(s) were ignored.", ignoredAmount);
        }
    }
    console.println();
    securityLogger.audit("Ingested {} file(s) from {}", ingestCount.get(), filePath);
    return null;
}

Example 2 with DigitalSignature

use of org.codice.ddf.commands.util.DigitalSignature in project ddf by codice.

the class DumpCommand method executeWithSubject.

@Override
protected final Object executeWithSubject() throws Exception {
    if (signer == null) {
        signer = new DigitalSignature(security);
    }
    if (FilenameUtils.getExtension(dirPath).equals("") && !dirPath.endsWith(File.separator)) {
        dirPath += File.separator;
    }
    final File dumpDir = new File(dirPath);
    if (!dumpDir.exists()) {
        printErrorMessage("Directory [" + dirPath + "] must exist.");
        console.println("If the directory does indeed exist, try putting the path in quotes.");
        return null;
    }
    if (!dumpDir.isDirectory()) {
        printErrorMessage("Path [" + dirPath + "] must be a directory.");
        return null;
    }
    if (!SERIALIZED_OBJECT_ID.matches(transformerId)) {
        transformers = getTransformers();
        if (transformers == null) {
            console.println(transformerId + " is an invalid metacard transformer.");
            return null;
        }
    }
    if (StringUtils.isNotBlank(zipFileName) && new File(dirPath + zipFileName).exists()) {
        console.println("Cannot dump Catalog.  Zip file " + zipFileName + " already exists.");
        return null;
    }
    if (StringUtils.isNotBlank(zipFileName) && !zipFileName.endsWith(".zip")) {
        zipFileName = zipFileName + ".zip";
    }
    securityLogger.audit("Called catalog:dump command with path : {}", dirPath);
    CatalogFacade catalog = getCatalog();
    SortBy sort = new SortByImpl(Core.ID, SortOrder.ASCENDING);
    QueryImpl query = new QueryImpl(getFilter());
    query.setRequestsTotalResultsCount(true);
    query.setPageSize(pageSize);
    query.setSortBy(sort);
    final AtomicLong resultCount = new AtomicLong(0);
    long start = System.currentTimeMillis();
    BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded);
    RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
    final ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, StandardThreadFactoryBuilder.newThreadFactory("dumpCommandThread"), rejectedExecutionHandler);
    QueryRequest queryRequest = new QueryRequestImpl(query);
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Hits for Search: {}", catalog.query(queryRequest).getHits());
    }
    if (StringUtils.isNotBlank(zipFileName)) {
        File outputFile = new File(dirPath + zipFileName);
        createZip(catalog, queryRequest, outputFile, resultCount);
        String alias = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty(SystemBaseUrl.EXTERNAL_HOST));
        String password = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty("javax.net.ssl.keyStorePassword"));
        try (InputStream inputStream = new FileInputStream(outputFile)) {
            byte[] signature = signer.createDigitalSignature(inputStream, alias, password);
            if (signature != null) {
                String epoch = Long.toString(Instant.now().getEpochSecond());
                String signatureFilepath = String.format("%sdump_%s.sig", dirPath, epoch);
                FileUtils.writeByteArrayToFile(new File(signatureFilepath), signature);
            }
        }
    } else {
        ResultIterable.resultIterable(catalog::query, queryRequest).stream().map(Collections::singletonList).map(result -> new SourceResponseImpl(queryRequest, result)).forEach(response -> handleResult(response, executorService, dumpDir, resultCount));
    }
    executorService.shutdown();
    boolean interrupted = false;
    try {
        while (!executorService.isTerminated()) {
            try {
                TimeUnit.MILLISECONDS.sleep(100);
            } catch (InterruptedException e) {
                interrupted = true;
            }
        }
    } finally {
        if (interrupted) {
            Thread.currentThread().interrupt();
        }
    }
    long end = System.currentTimeMillis();
    String elapsedTime = timeFormatter.print(new Period(start, end).withMillis(0));
    console.printf(" %d file(s) dumped in %s\t%n", resultCount.get(), elapsedTime);
    LOGGER.debug("{} file(s) dumped in {}", resultCount.get(), elapsedTime);
    console.println();
    securityLogger.audit("Exported {} files to {}", resultCount.get(), dirPath);
    return null;
}

Example 3 with DigitalSignature

use of org.codice.ddf.commands.util.DigitalSignature in project ddf by codice.

the class ExportCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    if (signer == null) {
        signer = new DigitalSignature(security);
    }
    Filter filter = getFilter();
    transformer = getServiceByFilter(MetacardTransformer.class, String.format("(%s=%s)", "id", DEFAULT_TRANSFORMER_ID)).orElseThrow(() -> new IllegalArgumentException("Could not get " + DEFAULT_TRANSFORMER_ID + " transformer"));
    revisionFilter = initRevisionFilter();
    final File outputFile = initOutputFile(output);
    checkFile(outputFile);
    if (delete && !force) {
        final String input = session.readLine("This action will remove all exported metacards and content from the catalog. Are you sure you wish to continue? (y/N):", null);
        if (input.length() == 0 || Character.toLowerCase(input.charAt(0)) != 'y') {
            console.println("ABORTED EXPORT.");
            return null;
        }
    }
    securityLogger.audit("Called catalog:export command with path : {}", output);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile);
        ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream)) {
        return doExport(outputFile, zipOutputStream, filter);
    } catch (FileNotFoundException e) {
        throw new FileNotFoundException(String.format("ZipOutputStream could not be created for the path %s", outputFile.getPath()));
    }
}

Example 4 with DigitalSignature

use of org.codice.ddf.commands.util.DigitalSignature in project ddf by codice.

the class ImportCommand method executeWithSubject.

@Override
protected final Object executeWithSubject() throws Exception {
    int metacards = 0;
    int content = 0;
    int derivedContent = 0;
    File file = initImportFile(importFile);
    InputTransformer transformer = getServiceByFilter(InputTransformer.class, String.format("(%s=%s)", "id", DEFAULT_TRANSFORMER_ID)).orElseThrow(() -> new CatalogCommandRuntimeException("Could not get " + DEFAULT_TRANSFORMER_ID + " input transformer"));
    if (unsafe) {
        if (!force) {
            String input = session.readLine("This will import data with no check to see if data is modified/corrupt. Do you wish to continue? (y/N) ", null);
            if (!input.matches("^[yY][eE]?[sS]?$")) {
                console.println("ABORTED IMPORT.");
                return null;
            }
        }
        securityLogger.audit("Skipping validation check of imported data. There are no " + "guarantees of integrity or authenticity of the imported data." + "File being imported: {}", importFile);
    } else {
        if (StringUtils.isBlank(signatureFile)) {
            String message = "A signature file must be provided with import data";
            console.println(message);
            throw new CatalogCommandRuntimeException(message);
        }
        String alias = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty("org.codice.ddf.system.hostname"));
        try (FileInputStream fileIs = new FileInputStream(file);
            FileInputStream sigFileIs = new FileInputStream(signatureFile)) {
            if (verifier == null) {
                verifier = new DigitalSignature(security);
            }
            if (!verifier.verifyDigitalSignature(fileIs, sigFileIs, alias)) {
                throw new CatalogCommandRuntimeException("The provided data could not be verified");
            }
        }
    }
    securityLogger.audit("Called catalog:import command on the file: {}", importFile);
    console.println("Importing file");
    Instant start = Instant.now();
    try (InputStream fis = new FileInputStream(file);
        ZipInputStream zipInputStream = new ZipInputStream(fis)) {
        ZipEntry entry = zipInputStream.getNextEntry();
        while (entry != null) {
            String filename = entry.getName();
            if (filename.startsWith("META-INF")) {
                entry = zipInputStream.getNextEntry();
                continue;
            }
            String[] pathParts = filename.split("\\" + File.separator);
            if (pathParts.length < 5) {
                console.println("Entry is not valid! " + filename);
                entry = zipInputStream.getNextEntry();
                continue;
            }
            String id = pathParts[ID];
            String type = pathParts[TYPE];
            switch(type) {
                case "metacard":
                    {
                        String metacardName = pathParts[NAME];
                        Metacard metacard = null;
                        try {
                            metacard = transformer.transform(new UncloseableBufferedInputStreamWrapper(zipInputStream), id);
                        } catch (IOException | CatalogTransformerException e) {
                            LOGGER.debug("Could not transform metacard: {}", LogSanitizer.sanitize(id));
                            entry = zipInputStream.getNextEntry();
                            continue;
                        }
                        metacard = applyInjectors(metacard, attributeInjectors);
                        catalogProvider.create(new CreateRequestImpl(metacard));
                        metacards++;
                        break;
                    }
                case "content":
                    {
                        content++;
                        String contentFilename = pathParts[NAME];
                        ContentItem contentItem = new ContentItemImpl(id, new ZipEntryByteSource(new UncloseableBufferedInputStreamWrapper(zipInputStream)), null, contentFilename, entry.getSize(), null);
                        CreateStorageRequestImpl createStorageRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>());
                        storageProvider.create(createStorageRequest);
                        storageProvider.commit(createStorageRequest);
                        break;
                    }
                case "derived":
                    {
                        derivedContent++;
                        String qualifier = pathParts[NAME];
                        String derivedContentName = pathParts[DERIVED_NAME];
                        ContentItem contentItem = new ContentItemImpl(id, qualifier, new ZipEntryByteSource(new UncloseableBufferedInputStreamWrapper(zipInputStream)), null, derivedContentName, entry.getSize(), null);
                        CreateStorageRequestImpl createStorageRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>());
                        storageProvider.create(createStorageRequest);
                        storageProvider.commit(createStorageRequest);
                        break;
                    }
                default:
                    {
                        LOGGER.debug("Cannot interpret type of {}", LogSanitizer.sanitize(type));
                    }
            }
            entry = zipInputStream.getNextEntry();
        }
    } catch (Exception e) {
        printErrorMessage(String.format("Exception while importing metacards (%s)%nFor more information set the log level to INFO (log:set INFO org.codice.ddf.commands.catalog) ", e.getMessage()));
        LOGGER.info("Exception while importing metacards", e);
        throw e;
    }
    console.println("File imported successfully. Imported in: " + getFormattedDuration(start));
    console.println("Number of metacards imported: " + metacards);
    console.println("Number of content imported: " + content);
    console.println("Number of derived content imported: " + derivedContent);
    return null;
}