use of org.codice.ddf.commands.catalog.export.ExportItem in project ddf by codice.
the class ExportCommand method doContentExport.
private List<ExportItem> doContentExport(/*Mutable,IO*/
ZipFile zipFile, List<ExportItem> exportedItems) throws ZipException {
List<ExportItem> contentItemsToExport = exportedItems.stream().filter(ei -> ei.getResourceUri() != null).filter(ei -> ei.getResourceUri().getScheme() != null).filter(ei -> ei.getResourceUri().getScheme().startsWith(ContentItem.CONTENT_SCHEME)).filter(ei -> !ei.getMetacardTag().equals("deleted")).filter(ei -> !ei.getMetacardTag().equals("revision") || ei.getResourceUri().getSchemeSpecificPart().equals(ei.getId())).filter(distinctByKey(ei -> ei.getResourceUri().getSchemeSpecificPart())).collect(Collectors.toList());
List<ExportItem> exportedContentItems = new ArrayList<>();
for (ExportItem contentItem : contentItemsToExport) {
ResourceResponse resource;
try {
resource = catalogFramework.getLocalResource(new ResourceRequestByProductUri(contentItem.getResourceUri()));
} catch (IOException | ResourceNotSupportedException e) {
throw new CatalogCommandRuntimeException("Unable to retrieve resource for " + contentItem.getId(), e);
} catch (ResourceNotFoundException e) {
continue;
}
writeToZip(zipFile, contentItem, resource);
exportedContentItems.add(contentItem);
if (!contentItem.getMetacardTag().equals("revision")) {
for (String derivedUri : contentItem.getDerivedUris()) {
URI uri;
try {
uri = new URI(derivedUri);
} catch (URISyntaxException e) {
LOGGER.debug("Uri [{}] is not a valid URI. Derived content will not be included in export", derivedUri);
continue;
}
ResourceResponse derivedResource;
try {
derivedResource = catalogFramework.getLocalResource(new ResourceRequestByProductUri(uri));
} catch (IOException e) {
throw new CatalogCommandRuntimeException("Unable to retrieve resource for " + contentItem.getId(), e);
} catch (ResourceNotFoundException | ResourceNotSupportedException e) {
LOGGER.warn("Could not retreive resource [{}]", uri, e);
console.printf("%sUnable to retrieve resource for export : %s%s%n", Ansi.ansi().fg(Ansi.Color.RED).toString(), uri, Ansi.ansi().reset().toString());
continue;
}
writeToZip(zipFile, contentItem, derivedResource);
}
}
}
return exportedContentItems;
}
use of org.codice.ddf.commands.catalog.export.ExportItem in project ddf by codice.
the class ExportCommand method executeWithSubject.
@Override
protected Object executeWithSubject() throws Exception {
Filter filter = getFilter();
transformer = getServiceByFilter(MetacardTransformer.class, String.format("(%s=%s)", "id", DEFAULT_TRANSFORMER_ID)).orElseThrow(() -> new CatalogCommandRuntimeException("Could not get " + DEFAULT_TRANSFORMER_ID + " transformer"));
revisionFilter = initRevisionFilter();
final File outputFile = initOutputFile(output);
if (outputFile.exists()) {
printErrorMessage(String.format("File [%s] already exists!", outputFile.getPath()));
return null;
}
final File parentDirectory = outputFile.getParentFile();
if (parentDirectory == null || !parentDirectory.isDirectory()) {
printErrorMessage(String.format("Directory [%s] must exist.", output));
console.println("If the directory does indeed exist, try putting the path in quotes.");
return null;
}
String filename = FilenameUtils.getName(outputFile.getPath());
if (StringUtils.isBlank(filename) || !filename.endsWith(".zip")) {
console.println("Filename must end with '.zip' and not be blank");
return null;
}
if (delete && !force) {
console.println("This action will remove all exported metacards and content from the catalog. Are you sure you wish to continue? (y/N):");
String input = getUserInputModifiable().toString();
if (!input.matches("^[yY][eE]?[sS]?$")) {
console.println("ABORTED EXPORT.");
return null;
}
}
SecurityLogger.audit("Called catalog:export command with path : {}", output);
ZipFile zipFile = new ZipFile(outputFile);
console.println("Starting metacard export...");
Instant start = Instant.now();
List<ExportItem> exportedItems = doMetacardExport(zipFile, filter);
console.println("Metacards exported in: " + getFormattedDuration(start));
console.println("Number of metacards exported: " + exportedItems.size());
console.println();
SecurityLogger.audit("Ids of exported metacards and content:\n{}", exportedItems.stream().map(ExportItem::getId).distinct().collect(Collectors.joining(", ", "[", "]")));
console.println("Starting content export...");
start = Instant.now();
List<ExportItem> exportedContentItems = doContentExport(zipFile, exportedItems);
console.println("Content exported in: " + getFormattedDuration(start));
console.println("Number of content exported: " + exportedContentItems.size());
console.println();
if (delete) {
doDelete(exportedItems, exportedContentItems);
}
if (!unsafe) {
SecurityLogger.audit("Signing exported data. file: [{}]", zipFile.getFile().getName());
console.println("Signing zip file...");
start = Instant.now();
jarSigner.signJar(zipFile.getFile(), System.getProperty("org.codice.ddf.system.hostname"), System.getProperty("javax.net.ssl.keyStorePassword"), System.getProperty("javax.net.ssl.keyStore"), System.getProperty("javax.net.ssl.keyStorePassword"));
console.println("zip file signed in: " + getFormattedDuration(start));
}
console.println("Export complete.");
console.println("Exported to: " + zipFile.getFile().getCanonicalPath());
return null;
}
use of org.codice.ddf.commands.catalog.export.ExportItem in project ddf by codice.
the class ExportCommand method doMetacardExport.
private List<ExportItem> doMetacardExport(/*Mutable,IO*/
ZipFile zipFile, Filter filter) {
Set<String> seenIds = new HashSet<>(1024);
List<ExportItem> exportedItems = new ArrayList<>();
for (Result result : new QueryResultIterable(catalogFramework, (i) -> getQuery(filter, i, PAGE_SIZE), PAGE_SIZE)) {
if (!seenIds.contains(result.getMetacard().getId())) {
writeToZip(zipFile, result);
exportedItems.add(new ExportItem(result.getMetacard().getId(), getTag(result), result.getMetacard().getResourceURI(), getDerivedResources(result)));
seenIds.add(result.getMetacard().getId());
}
// Fetch and export all history for each exported item
for (Result revision : new QueryResultIterable(catalogFramework, (i) -> getQuery(getHistoryFilter(result), i, PAGE_SIZE), PAGE_SIZE)) {
if (seenIds.contains(revision.getMetacard().getId())) {
continue;
}
writeToZip(zipFile, revision);
exportedItems.add(new ExportItem(revision.getMetacard().getId(), getTag(revision), revision.getMetacard().getResourceURI(), getDerivedResources(result)));
seenIds.add(revision.getMetacard().getId());
}
}
return exportedItems;
}
use of org.codice.ddf.commands.catalog.export.ExportItem in project ddf by codice.
the class ExportCommand method doDelete.
private void doDelete(List<ExportItem> exportedItems, List<ExportItem> exportedContentItems) {
Instant start;
console.println("Starting delete");
start = Instant.now();
for (ExportItem exportedContentItem : exportedContentItems) {
try {
DeleteStorageRequestImpl deleteRequest = new DeleteStorageRequestImpl(Collections.singletonList(new IdAndUriMetacard(exportedContentItem.getId(), exportedContentItem.getResourceUri())), exportedContentItem.getId(), Collections.emptyMap());
storageProvider.delete(deleteRequest);
storageProvider.commit(deleteRequest);
} catch (StorageException e) {
printErrorMessage("Could not delete content for metacard: " + exportedContentItem.toString());
}
}
for (ExportItem exported : exportedItems) {
try {
catalogProvider.delete(new DeleteRequestImpl(exported.getId()));
} catch (IngestException e) {
printErrorMessage("Could not delete metacard: " + exported.toString());
}
}
// delete items from cache
try {
getCacheProxy().removeById(exportedItems.stream().map(ExportItem::getId).collect(Collectors.toList()).toArray(new String[exportedItems.size()]));
} catch (Exception e) {
LOGGER.warn("Could not delete all exported items from cache (Results will eventually expire)", e);
}
console.println("Metacards and Content deleted in: " + getFormattedDuration(start));
console.println("Number of metacards deleted: " + exportedItems.size());
console.println("Number of content deleted: " + exportedContentItems.size());
}
Aggregations