Search in sources :

Example 1 with ExportItem

use of org.codice.ddf.commands.catalog.export.ExportItem in project ddf by codice.

the class ExportCommand method doContentExport.

private List<ExportItem> doContentExport(/*Mutable,IO*/
ZipFile zipFile, List<ExportItem> exportedItems) throws ZipException {
    List<ExportItem> contentItemsToExport = exportedItems.stream().filter(ei -> ei.getResourceUri() != null).filter(ei -> ei.getResourceUri().getScheme() != null).filter(ei -> ei.getResourceUri().getScheme().startsWith(ContentItem.CONTENT_SCHEME)).filter(ei -> !ei.getMetacardTag().equals("deleted")).filter(ei -> !ei.getMetacardTag().equals("revision") || ei.getResourceUri().getSchemeSpecificPart().equals(ei.getId())).filter(distinctByKey(ei -> ei.getResourceUri().getSchemeSpecificPart())).collect(Collectors.toList());
    List<ExportItem> exportedContentItems = new ArrayList<>();
    for (ExportItem contentItem : contentItemsToExport) {
        ResourceResponse resource;
        try {
            resource = catalogFramework.getLocalResource(new ResourceRequestByProductUri(contentItem.getResourceUri()));
        } catch (IOException | ResourceNotSupportedException e) {
            throw new CatalogCommandRuntimeException("Unable to retrieve resource for " + contentItem.getId(), e);
        } catch (ResourceNotFoundException e) {
            continue;
        }
        writeToZip(zipFile, contentItem, resource);
        exportedContentItems.add(contentItem);
        if (!contentItem.getMetacardTag().equals("revision")) {
            for (String derivedUri : contentItem.getDerivedUris()) {
                URI uri;
                try {
                    uri = new URI(derivedUri);
                } catch (URISyntaxException e) {
                    LOGGER.debug("Uri [{}] is not a valid URI. Derived content will not be included in export", derivedUri);
                    continue;
                }
                ResourceResponse derivedResource;
                try {
                    derivedResource = catalogFramework.getLocalResource(new ResourceRequestByProductUri(uri));
                } catch (IOException e) {
                    throw new CatalogCommandRuntimeException("Unable to retrieve resource for " + contentItem.getId(), e);
                } catch (ResourceNotFoundException | ResourceNotSupportedException e) {
                    LOGGER.warn("Could not retreive resource [{}]", uri, e);
                    console.printf("%sUnable to retrieve resource for export : %s%s%n", Ansi.ansi().fg(Ansi.Color.RED).toString(), uri, Ansi.ansi().reset().toString());
                    continue;
                }
                writeToZip(zipFile, contentItem, derivedResource);
            }
        }
    }
    return exportedContentItems;
}
Also used : Ansi(org.fusesource.jansi.Ansi) StringUtils(org.apache.commons.lang.StringUtils) DeleteStorageRequestImpl(ddf.catalog.content.operation.impl.DeleteStorageRequestImpl) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) BinaryContent(ddf.catalog.data.BinaryContent) LoggerFactory(org.slf4j.LoggerFactory) SecurityLogger(ddf.security.common.audit.SecurityLogger) Command(org.apache.karaf.shell.api.action.Command) MetacardTransformer(ddf.catalog.transform.MetacardTransformer) MetacardVersion(ddf.catalog.core.versioning.MetacardVersion) Map(java.util.Map) ExportItem(org.codice.ddf.commands.catalog.export.ExportItem) DeleteRequestImpl(ddf.catalog.operation.impl.DeleteRequestImpl) URI(java.net.URI) ParseException(java.text.ParseException) ZipFile(net.lingala.zip4j.core.ZipFile) TimeZone(java.util.TimeZone) Predicate(java.util.function.Predicate) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) DeletedMetacard(ddf.catalog.core.versioning.DeletedMetacard) Set(java.util.Set) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) ResourceNotFoundException(ddf.catalog.resource.ResourceNotFoundException) CatalogTransformerException(ddf.catalog.transform.CatalogTransformerException) Objects(java.util.Objects) ZipException(net.lingala.zip4j.exception.ZipException) StorageException(ddf.catalog.content.StorageException) List(java.util.List) FilenameUtils(org.apache.commons.io.FilenameUtils) ResourceResponse(ddf.catalog.operation.ResourceResponse) QueryRequestImpl(ddf.catalog.operation.impl.QueryRequestImpl) SimpleDateFormat(java.text.SimpleDateFormat) CatalogCommandRuntimeException(org.codice.ddf.commands.util.CatalogCommandRuntimeException) HashMap(java.util.HashMap) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Reference(org.apache.karaf.shell.api.action.lifecycle.Reference) SortBy(org.opengis.filter.sort.SortBy) ContentItem(ddf.catalog.content.data.ContentItem) Metacard(ddf.catalog.data.Metacard) CQLException(org.geotools.filter.text.cql2.CQLException) StorageProvider(ddf.catalog.content.StorageProvider) Result(ddf.catalog.data.Result) QueryImpl(ddf.catalog.operation.impl.QueryImpl) Logger(org.slf4j.Logger) ZipParameters(net.lingala.zip4j.model.ZipParameters) QueryResultIterable(org.codice.ddf.commands.util.QueryResultIterable) IngestException(ddf.catalog.source.IngestException) ResourceRequestByProductUri(ddf.catalog.operation.impl.ResourceRequestByProductUri) IOException(java.io.IOException) IdAndUriMetacard(org.codice.ddf.commands.catalog.export.IdAndUriMetacard) File(java.io.File) JarSigner(org.codice.ddf.catalog.transformer.zip.JarSigner) TimeUnit(java.util.concurrent.TimeUnit) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException) Paths(java.nio.file.Paths) Service(org.apache.karaf.shell.api.action.lifecycle.Service) Filter(org.opengis.filter.Filter) Option(org.apache.karaf.shell.api.action.Option) Collections(java.util.Collections) ExportItem(org.codice.ddf.commands.catalog.export.ExportItem) ArrayList(java.util.ArrayList) CatalogCommandRuntimeException(org.codice.ddf.commands.util.CatalogCommandRuntimeException) IOException(java.io.IOException) URISyntaxException(java.net.URISyntaxException) ResourceRequestByProductUri(ddf.catalog.operation.impl.ResourceRequestByProductUri) URI(java.net.URI) ResourceResponse(ddf.catalog.operation.ResourceResponse) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException) ResourceNotFoundException(ddf.catalog.resource.ResourceNotFoundException)

Example 2 with ExportItem

use of org.codice.ddf.commands.catalog.export.ExportItem in project ddf by codice.

the class ExportCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    Filter filter = getFilter();
    transformer = getServiceByFilter(MetacardTransformer.class, String.format("(%s=%s)", "id", DEFAULT_TRANSFORMER_ID)).orElseThrow(() -> new CatalogCommandRuntimeException("Could not get " + DEFAULT_TRANSFORMER_ID + " transformer"));
    revisionFilter = initRevisionFilter();
    final File outputFile = initOutputFile(output);
    if (outputFile.exists()) {
        printErrorMessage(String.format("File [%s] already exists!", outputFile.getPath()));
        return null;
    }
    final File parentDirectory = outputFile.getParentFile();
    if (parentDirectory == null || !parentDirectory.isDirectory()) {
        printErrorMessage(String.format("Directory [%s] must exist.", output));
        console.println("If the directory does indeed exist, try putting the path in quotes.");
        return null;
    }
    String filename = FilenameUtils.getName(outputFile.getPath());
    if (StringUtils.isBlank(filename) || !filename.endsWith(".zip")) {
        console.println("Filename must end with '.zip' and not be blank");
        return null;
    }
    if (delete && !force) {
        console.println("This action will remove all exported metacards and content from the catalog. Are you sure you wish to continue? (y/N):");
        String input = getUserInputModifiable().toString();
        if (!input.matches("^[yY][eE]?[sS]?$")) {
            console.println("ABORTED EXPORT.");
            return null;
        }
    }
    SecurityLogger.audit("Called catalog:export command with path : {}", output);
    ZipFile zipFile = new ZipFile(outputFile);
    console.println("Starting metacard export...");
    Instant start = Instant.now();
    List<ExportItem> exportedItems = doMetacardExport(zipFile, filter);
    console.println("Metacards exported in: " + getFormattedDuration(start));
    console.println("Number of metacards exported: " + exportedItems.size());
    console.println();
    SecurityLogger.audit("Ids of exported metacards and content:\n{}", exportedItems.stream().map(ExportItem::getId).distinct().collect(Collectors.joining(", ", "[", "]")));
    console.println("Starting content export...");
    start = Instant.now();
    List<ExportItem> exportedContentItems = doContentExport(zipFile, exportedItems);
    console.println("Content exported in: " + getFormattedDuration(start));
    console.println("Number of content exported: " + exportedContentItems.size());
    console.println();
    if (delete) {
        doDelete(exportedItems, exportedContentItems);
    }
    if (!unsafe) {
        SecurityLogger.audit("Signing exported data. file: [{}]", zipFile.getFile().getName());
        console.println("Signing zip file...");
        start = Instant.now();
        jarSigner.signJar(zipFile.getFile(), System.getProperty("org.codice.ddf.system.hostname"), System.getProperty("javax.net.ssl.keyStorePassword"), System.getProperty("javax.net.ssl.keyStore"), System.getProperty("javax.net.ssl.keyStorePassword"));
        console.println("zip file signed in: " + getFormattedDuration(start));
    }
    console.println("Export complete.");
    console.println("Exported to: " + zipFile.getFile().getCanonicalPath());
    return null;
}
Also used : ExportItem(org.codice.ddf.commands.catalog.export.ExportItem) ZipFile(net.lingala.zip4j.core.ZipFile) Filter(org.opengis.filter.Filter) Instant(java.time.Instant) CatalogCommandRuntimeException(org.codice.ddf.commands.util.CatalogCommandRuntimeException) ZipFile(net.lingala.zip4j.core.ZipFile) File(java.io.File)

Example 3 with ExportItem

use of org.codice.ddf.commands.catalog.export.ExportItem in project ddf by codice.

the class ExportCommand method doMetacardExport.

private List<ExportItem> doMetacardExport(/*Mutable,IO*/
ZipFile zipFile, Filter filter) {
    Set<String> seenIds = new HashSet<>(1024);
    List<ExportItem> exportedItems = new ArrayList<>();
    for (Result result : new QueryResultIterable(catalogFramework, (i) -> getQuery(filter, i, PAGE_SIZE), PAGE_SIZE)) {
        if (!seenIds.contains(result.getMetacard().getId())) {
            writeToZip(zipFile, result);
            exportedItems.add(new ExportItem(result.getMetacard().getId(), getTag(result), result.getMetacard().getResourceURI(), getDerivedResources(result)));
            seenIds.add(result.getMetacard().getId());
        }
        // Fetch and export all history for each exported item
        for (Result revision : new QueryResultIterable(catalogFramework, (i) -> getQuery(getHistoryFilter(result), i, PAGE_SIZE), PAGE_SIZE)) {
            if (seenIds.contains(revision.getMetacard().getId())) {
                continue;
            }
            writeToZip(zipFile, revision);
            exportedItems.add(new ExportItem(revision.getMetacard().getId(), getTag(revision), revision.getMetacard().getResourceURI(), getDerivedResources(result)));
            seenIds.add(revision.getMetacard().getId());
        }
    }
    return exportedItems;
}
Also used : QueryResultIterable(org.codice.ddf.commands.util.QueryResultIterable) ExportItem(org.codice.ddf.commands.catalog.export.ExportItem) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Result(ddf.catalog.data.Result)

Example 4 with ExportItem

use of org.codice.ddf.commands.catalog.export.ExportItem in project ddf by codice.

the class ExportCommand method doDelete.

private void doDelete(List<ExportItem> exportedItems, List<ExportItem> exportedContentItems) {
    Instant start;
    console.println("Starting delete");
    start = Instant.now();
    for (ExportItem exportedContentItem : exportedContentItems) {
        try {
            DeleteStorageRequestImpl deleteRequest = new DeleteStorageRequestImpl(Collections.singletonList(new IdAndUriMetacard(exportedContentItem.getId(), exportedContentItem.getResourceUri())), exportedContentItem.getId(), Collections.emptyMap());
            storageProvider.delete(deleteRequest);
            storageProvider.commit(deleteRequest);
        } catch (StorageException e) {
            printErrorMessage("Could not delete content for metacard: " + exportedContentItem.toString());
        }
    }
    for (ExportItem exported : exportedItems) {
        try {
            catalogProvider.delete(new DeleteRequestImpl(exported.getId()));
        } catch (IngestException e) {
            printErrorMessage("Could not delete metacard: " + exported.toString());
        }
    }
    // delete items from cache
    try {
        getCacheProxy().removeById(exportedItems.stream().map(ExportItem::getId).collect(Collectors.toList()).toArray(new String[exportedItems.size()]));
    } catch (Exception e) {
        LOGGER.warn("Could not delete all exported items from cache (Results will eventually expire)", e);
    }
    console.println("Metacards and Content deleted in: " + getFormattedDuration(start));
    console.println("Number of metacards deleted: " + exportedItems.size());
    console.println("Number of content deleted: " + exportedContentItems.size());
}
Also used : DeleteStorageRequestImpl(ddf.catalog.content.operation.impl.DeleteStorageRequestImpl) ExportItem(org.codice.ddf.commands.catalog.export.ExportItem) Instant(java.time.Instant) DeleteRequestImpl(ddf.catalog.operation.impl.DeleteRequestImpl) IdAndUriMetacard(org.codice.ddf.commands.catalog.export.IdAndUriMetacard) IngestException(ddf.catalog.source.IngestException) StorageException(ddf.catalog.content.StorageException) URISyntaxException(java.net.URISyntaxException) ParseException(java.text.ParseException) ResourceNotFoundException(ddf.catalog.resource.ResourceNotFoundException) CatalogTransformerException(ddf.catalog.transform.CatalogTransformerException) ZipException(net.lingala.zip4j.exception.ZipException) StorageException(ddf.catalog.content.StorageException) CatalogCommandRuntimeException(org.codice.ddf.commands.util.CatalogCommandRuntimeException) CQLException(org.geotools.filter.text.cql2.CQLException) IngestException(ddf.catalog.source.IngestException) IOException(java.io.IOException) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException)

Aggregations

Instant (java.time.Instant)3 ExportItem (org.codice.ddf.commands.catalog.export.ExportItem)3 StorageException (ddf.catalog.content.StorageException)2 DeleteStorageRequestImpl (ddf.catalog.content.operation.impl.DeleteStorageRequestImpl)2 Result (ddf.catalog.data.Result)2 DeleteRequestImpl (ddf.catalog.operation.impl.DeleteRequestImpl)2 ResourceNotFoundException (ddf.catalog.resource.ResourceNotFoundException)2 ResourceNotSupportedException (ddf.catalog.resource.ResourceNotSupportedException)2 IngestException (ddf.catalog.source.IngestException)2 CatalogTransformerException (ddf.catalog.transform.CatalogTransformerException)2 File (java.io.File)2 IOException (java.io.IOException)2 URISyntaxException (java.net.URISyntaxException)2 ParseException (java.text.ParseException)2 ArrayList (java.util.ArrayList)2 HashSet (java.util.HashSet)2 CatalogCommandRuntimeException (org.codice.ddf.commands.util.CatalogCommandRuntimeException)2 StorageProvider (ddf.catalog.content.StorageProvider)1 ContentItem (ddf.catalog.content.data.ContentItem)1 DeletedMetacard (ddf.catalog.core.versioning.DeletedMetacard)1