Examples with ExportTemplate com.thinkbiganalytics.metadata.api.template.export.ExportTemplate used on opensource projects

Example 1 with ExportTemplate

use of com.thinkbiganalytics.metadata.api.template.export.ExportTemplate in project kylo by Teradata.

the class FilesystemRepositoryService method publishTemplate.

@Override
public TemplateMetadataWrapper publishTemplate(String repositoryName, String repositoryType, String templateId, boolean overwrite) throws Exception {
    // export template
    ExportTemplate zipFile = templateExporter.exportTemplate(templateId);
    // get repository
    TemplateRepository repository = getRepositoryByNameAndType(repositoryName, repositoryType);
    // Check if template already exists
    Optional<TemplateMetadataWrapper> foundMetadataOpt = Optional.empty();
    for (TemplateMetadataWrapper item : listTemplatesByRepository(repository)) {
        if (item.getTemplateName().equals(zipFile.getTemplateName())) {
            foundMetadataOpt = Optional.of(item);
            break;
        }
    }
    if (foundMetadataOpt.isPresent()) {
        if (!overwrite) {
            throw new UnsupportedOperationException("Template with same name already exists.");
        }
        log.info("Overwriting template with same name.");
    }
    String digest = DigestUtils.md5DigestAsHex(zipFile.getFile());
    Path templatePath = Paths.get(repository.getLocation() + "/" + zipFile.getFileName());
    TemplateMetadata metadata = new TemplateMetadata(zipFile.getTemplateName(), zipFile.getDescription(), zipFile.getFileName().toString(), digest, zipFile.isStream(), false, templatePath.toFile().lastModified());
    // create repositoryItem
    TemplateMetadataWrapper templateMetadata = foundMetadataOpt.isPresent() ? foundMetadataOpt.get() : new TemplateMetadataWrapper(metadata);
    String baseName = FilenameUtils.getBaseName(templateMetadata.getFileName());
    // write file in first of templateLocations
    Files.write(templatePath, zipFile.getFile());
    log.info("Finished publishing template {} to repository {}.", templateMetadata.getTemplateName(), repository.getName());
    return new TemplateMetadataWrapper(metadata);
}

Example 2 with ExportTemplate

use of com.thinkbiganalytics.metadata.api.template.export.ExportTemplate in project kylo by Teradata.

the class FilesystemRepositoryServiceTest method testPublishTemplate_NewTemplateInRepo_NoOverwriteRequired.

@Test
public void testPublishTemplate_NewTemplateInRepo_NoOverwriteRequired() throws Exception {
    ExportTemplate template = new ExportTemplate("testFile", "testTemplate", "testDesc", false, new byte[0]);
    when(templateExporter.exportTemplate(anyString())).thenReturn(template);
    FilesystemRepositoryService spy = PowerMockito.spy(filesystemRepositoryService);
    doReturn(repository).when(spy, "getRepositoryByNameAndType", anyString(), anyString());
    doReturn(new ArrayList<TemplateMetadata>()).when(spy, "listTemplatesByRepository", repository);
    PowerMockito.when(FilenameUtils.getBaseName(anyString())).thenReturn("testName");
    Path pathMock = mock(Path.class);
    PowerMockito.when(Paths.get(anyString())).thenReturn(pathMock);
    Mockito.when(pathMock.toFile()).thenReturn(mock(File.class));
    Mockito.doNothing().when(mapper).writeValue(any(File.class), any());
    TemplateMetadataWrapper newTemplate = spy.publishTemplate("test", "test", "test", false);
    assertNotNull(newTemplate);
    assertTrue(newTemplate.getTemplateName().equals("testTemplate"));
}

Example 3 with ExportTemplate

use of com.thinkbiganalytics.metadata.api.template.export.ExportTemplate in project kylo by Teradata.

the class FilesystemRepositoryServiceTest method testPublishTemplate_ExistingTemplate_OverwriteFalse.

@Test(expected = UnsupportedOperationException.class)
public void testPublishTemplate_ExistingTemplate_OverwriteFalse() throws Exception {
    ExportTemplate template = new ExportTemplate("testFile", "testTemplate", "testDesc", false, new byte[0]);
    TemplateMetadata existingTemplate = new TemplateMetadata("testTemplate", "testDesc", "xyz.zip", false);
    List<TemplateMetadataWrapper> templates = new ArrayList<>();
    templates.add(new TemplateMetadataWrapper(existingTemplate));
    when(templateExporter.exportTemplate(anyString())).thenReturn(template);
    FilesystemRepositoryService spy = PowerMockito.spy(filesystemRepositoryService);
    doReturn(repository).when(spy, "getRepositoryByNameAndType", anyString(), anyString());
    doReturn(templates).when(spy, "listTemplatesByRepository", repository);
    spy.publishTemplate("test", "test", "test", false);
    PowerMockito.verifyStatic(never());
    FilenameUtils.getBaseName(anyString());
}

Example 4 with ExportTemplate

use of com.thinkbiganalytics.metadata.api.template.export.ExportTemplate in project kylo by Teradata.

the class RepositoryMonitor method createMetadata.

private void createMetadata(Path templateFilePath, Map<String, RegisteredTemplate> registeredTemplateMap) {
    RegisteredTemplate tmplt = null;
    try {
        File templateZipFile = templateFilePath.toFile();
        byte[] content = ImportUtil.streamToByteArray(new FileInputStream(templateZipFile));
        InputStream inputStream = new ByteArrayInputStream(content);
        ImportTemplate importTemplate = ImportUtil.openZip(templateFilePath.getFileName().toString(), inputStream);
        tmplt = importTemplate.getTemplateToImport();
        File json = getMetadataFileName(templateFilePath).toFile();
        String checksum = DigestUtils.md5DigestAsHex(content);
        boolean updateAvailable = false;
        TemplateMetadata metadata = new TemplateMetadata(tmplt.getTemplateName(), tmplt.getDescription(), templateFilePath.getFileName().toString(), checksum, tmplt.isStream(), updateAvailable, tmplt.getUpdateDate().getTime());
        // startup or new template is published
        if (!json.exists()) {
            if (registeredTemplateMap.containsKey(tmplt.getTemplateName())) {
                final String templateName = tmplt.getTemplateName();
                ExportTemplate zipFile = metadataAccess.read(() -> {
                    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_TEMPLATES);
                    return templateExporter.exportTemplate(registeredTemplateMap.get(templateName).getId());
                }, MetadataAccess.SERVICE);
                String digest = DigestUtils.md5DigestAsHex(zipFile.getFile());
                updateAvailable = !StringUtils.equals(checksum, digest);
                metadata.setUpdateAvailable(updateAvailable);
                templateUpdateInfoCache.put(templateName, updateAvailable);
            }
        } else {
            // if registered template was updated
            metadata = mapper.readValue(json, TemplateMetadata.class);
            templateUpdateInfoCache.put(tmplt.getTemplateName(), metadata.isUpdateAvailable());
            if (metadata.getLastModified() >= tmplt.getUpdateDate().getTime()) {
                return;
            }
            if (StringUtils.equals(metadata.getChecksum(), checksum)) {
                return;
            }
            metadata.setUpdateAvailable(true);
            templateUpdateInfoCache.put(tmplt.getTemplateName(), true);
        }
        log.info("Writing template metadata for {}.", tmplt.getTemplateName());
        mapper.writerWithDefaultPrettyPrinter().writeValue(json, metadata);
    } catch (Exception e) {
        log.error("Error occurred trying to generate template metadata.", e);
        if (tmplt != null) {
            templateUpdateInfoCache.invalidate(tmplt.getTemplateName());
        }
    }
}

Example 5 with ExportTemplate

use of com.thinkbiganalytics.metadata.api.template.export.ExportTemplate in project kylo by Teradata.

the class DefaultFeedExporter method exportFeed.

/**
 * Export a feed as a zip file
 *
 * @param feedId the id {@link Feed#getId()} of the feed to export
 * @return object containing the zip file with data about the feed.
 */
@Override
public ExportFeed exportFeed(String feedId) throws IOException {
    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EXPORT_FEEDS);
    this.metadataService.checkFeedPermission(feedId, FeedAccessControl.EXPORT);
    // Prepare feed metadata
    final FeedMetadata feed = metadataService.getFeedById(feedId);
    if (feed == null) {
        // feed will not be found when user is allowed to export feeds but has no entity access to feed with feed id
        throw new NotFoundException("Feed not found for id " + feedId);
    }
    /*
        final List<Datasource> userDatasources = Optional.ofNullable(feed.getDataTransformation())
            .map(FeedDataTransformation::getDatasourceIds)
            .map(datasourceIds -> metadataAccess.read(
                () ->
                    datasourceIds.stream()
                        .map(datasourceProvider::resolve)
                        .map(datasourceProvider::getDatasource)
                        .map(domain -> datasourceTransform.toDatasource(domain, DatasourceModelTransform.Level.FULL))
                        .map(datasource -> {
                            // Clear sensitive fields
                            datasource.getDestinationForFeeds().clear();
                            datasource.getSourceForFeeds().clear();
                            return datasource;
                        })
                        .collect(Collectors.toList())
                 )
            )
            .orElse(null);
        if (userDatasources != null && !userDatasources.isEmpty()) {
            this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
            feed.setUserDatasources(userDatasources);
        }
        */
    // export the datasets ???
    String dataSetJSON = "[]";
    if (!feed.getSourceDataSets().isEmpty()) {
        // refetch them?
        List<com.thinkbiganalytics.kylo.catalog.rest.model.DataSet> dataSets = feed.getSourceDataSets().stream().map(dataSet -> {
            return metadataAccess.read(() -> {
                return dataSetProvider.find(dataSetProvider.resolveId(dataSet.getId())).map(ds -> catalogModelTransform.dataSetToRestModel().apply(ds)).orElse(null);
            });
        }).filter(dataSet -> dataSet != null).collect(Collectors.toList());
        dataSetJSON = ObjectMapperSerializer.serialize(dataSets);
    }
    // Add feed json to template zip file
    final ExportTemplate exportTemplate = templateExporter.exportTemplateForFeedExport(feed.getTemplateId());
    final String feedJson = ObjectMapperSerializer.serialize(feed);
    byte[] zipFile = ZipFileUtil.addToZip(exportTemplate.getFile(), feedJson, ImportFeed.FEED_JSON_FILE);
    final byte[] zipFile2 = ZipFileUtil.addToZip(zipFile, dataSetJSON, ImportFeed.FEED_DATASETS_FILE);
    return new ExportFeed(feed.getSystemFeedName() + ".feed.zip", zipFile2);
}