Examples with TemplateMetadata com.thinkbiganalytics.repository.api.TemplateMetadata used on opensource projects

Example 1 with TemplateMetadata

use of com.thinkbiganalytics.repository.api.TemplateMetadata in project kylo by Teradata.

the class FilesystemRepositoryService method jsonToMetadata.

private TemplateMetadataWrapper jsonToMetadata(Path path, TemplateRepository repository, Optional<Set<String>> registeredTemplates) {
    try {
        TemplateMetadata templateMetaData = mapper.readValue(path.toFile(), TemplateMetadata.class);
        TemplateMetadataWrapper wrapper = new TemplateMetadataWrapper(templateMetaData);
        Path templatePath = Paths.get(repository.getLocation() + "/" + templateMetaData.getFileName());
        if (registeredTemplates.isPresent() && registeredTemplates.get().contains(templateMetaData.getTemplateName())) {
            byte[] content = ImportUtil.streamToByteArray(new FileInputStream(templatePath.toFile()));
            InputStream inputStream = new ByteArrayInputStream(content);
            ImportTemplate importTemplate = ImportUtil.openZip(templatePath.getFileName().toString(), inputStream);
            RegisteredTemplate template = importTemplate.getTemplateToImport();
            templateUpdateInfoCache.get(templateMetaData.getTemplateName(), () -> templateMetaData.isUpdateAvailable());
            // set updated flag if file is modified.
            if (templateMetaData.getLastModified() < template.getUpdateDate().getTime()) {
                String checksum = DigestUtils.md5DigestAsHex(content);
                // capturing checksum first time or it has actually changed?
                boolean templateModified = !StringUtils.equals(templateMetaData.getChecksum(), checksum);
                templateMetaData.setUpdateAvailable(templateModified);
                templateUpdateInfoCache.put(templateMetaData.getTemplateName(), templateModified);
                mapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), templateMetaData);
                wrapper = new TemplateMetadataWrapper(templateMetaData);
            }
            if (wrapper.isUpdateAvailable()) {
                wrapper.getUpdates().addAll(template.getChangeComments());
            }
            wrapper.setInstalled(true);
        }
        wrapper.setRepository(repository);
        return wrapper;
    } catch (Exception e) {
        log.error("Error reading metadata", e);
    }
    return null;
}

Example 2 with TemplateMetadata

use of com.thinkbiganalytics.repository.api.TemplateMetadata in project kylo by Teradata.

the class FilesystemRepositoryService method publishTemplate.

@Override
public TemplateMetadataWrapper publishTemplate(String repositoryName, String repositoryType, String templateId, boolean overwrite) throws Exception {
    // export template
    ExportTemplate zipFile = templateExporter.exportTemplate(templateId);
    // get repository
    TemplateRepository repository = getRepositoryByNameAndType(repositoryName, repositoryType);
    // Check if template already exists
    Optional<TemplateMetadataWrapper> foundMetadataOpt = Optional.empty();
    for (TemplateMetadataWrapper item : listTemplatesByRepository(repository)) {
        if (item.getTemplateName().equals(zipFile.getTemplateName())) {
            foundMetadataOpt = Optional.of(item);
            break;
        }
    }
    if (foundMetadataOpt.isPresent()) {
        if (!overwrite) {
            throw new UnsupportedOperationException("Template with same name already exists.");
        }
        log.info("Overwriting template with same name.");
    }
    String digest = DigestUtils.md5DigestAsHex(zipFile.getFile());
    Path templatePath = Paths.get(repository.getLocation() + "/" + zipFile.getFileName());
    TemplateMetadata metadata = new TemplateMetadata(zipFile.getTemplateName(), zipFile.getDescription(), zipFile.getFileName().toString(), digest, zipFile.isStream(), false, templatePath.toFile().lastModified());
    // create repositoryItem
    TemplateMetadataWrapper templateMetadata = foundMetadataOpt.isPresent() ? foundMetadataOpt.get() : new TemplateMetadataWrapper(metadata);
    String baseName = FilenameUtils.getBaseName(templateMetadata.getFileName());
    // write file in first of templateLocations
    Files.write(templatePath, zipFile.getFile());
    log.info("Finished publishing template {} to repository {}.", templateMetadata.getTemplateName(), repository.getName());
    return new TemplateMetadataWrapper(metadata);
}

Example 3 with TemplateMetadata

use of com.thinkbiganalytics.repository.api.TemplateMetadata in project kylo by Teradata.

the class FilesystemRepositoryServiceTest method testPublishTemplate_ExistingTemplate_OverwriteFalse.

@Test(expected = UnsupportedOperationException.class)
public void testPublishTemplate_ExistingTemplate_OverwriteFalse() throws Exception {
    ExportTemplate template = new ExportTemplate("testFile", "testTemplate", "testDesc", false, new byte[0]);
    TemplateMetadata existingTemplate = new TemplateMetadata("testTemplate", "testDesc", "xyz.zip", false);
    List<TemplateMetadataWrapper> templates = new ArrayList<>();
    templates.add(new TemplateMetadataWrapper(existingTemplate));
    when(templateExporter.exportTemplate(anyString())).thenReturn(template);
    FilesystemRepositoryService spy = PowerMockito.spy(filesystemRepositoryService);
    doReturn(repository).when(spy, "getRepositoryByNameAndType", anyString(), anyString());
    doReturn(templates).when(spy, "listTemplatesByRepository", repository);
    spy.publishTemplate("test", "test", "test", false);
    PowerMockito.verifyStatic(never());
    FilenameUtils.getBaseName(anyString());
}

Example 4 with TemplateMetadata

use of com.thinkbiganalytics.repository.api.TemplateMetadata in project kylo by Teradata.

the class FilesystemRepositoryService method importTemplate.

@Override
public ImportTemplate importTemplate(String repositoryName, String repositoryType, String fileName, String uploadKey, String importComponents) throws Exception {
    Optional<TemplateRepository> repository = listRepositories().stream().filter(r -> StringUtils.equals(r.getName(), repositoryName) && StringUtils.equals(r.getType().getKey(), repositoryType)).findFirst();
    Path filePath = Paths.get(repository.get().getLocation() + "/" + fileName);
    if (!filePath.toFile().exists()) {
        throw new RuntimeException("Unable to find template file to import: " + fileName);
    }
    log.info("Begin template import {}", fileName);
    ImportTemplateOptions options = new ImportTemplateOptions();
    options.setUploadKey(uploadKey);
    byte[] content = ImportUtil.streamToByteArray(new FileInputStream(filePath.toFile()));
    String checksum = DigestUtils.md5DigestAsHex(content);
    uploadProgressService.newUpload(uploadKey);
    ImportTemplate importTemplate = null;
    TemplateImporter templateImporter = null;
    if (importComponents == null) {
        templateImporter = templateImporterFactory.apply(fileName, content, options);
        importTemplate = templateImporter.validate();
        importTemplate.setSuccess(false);
    } else {
        options.setImportComponentOptions(ObjectMapperSerializer.deserialize(importComponents, new TypeReference<Set<ImportComponentOption>>() {
        }));
        templateImporter = templateImporterFactory.apply(fileName, content, options);
        importTemplate = templateImporter.validateAndImport();
    }
    log.info("End template import {} - {}", fileName, importTemplate.isSuccess());
    // update template metadata
    String baseName = FilenameUtils.getBaseName(fileName);
    Path metadataPath = Paths.get(repository.get().getLocation() + "/" + baseName + ".json");
    TemplateMetadata templateMetadata = mapper.readValue(metadataPath.toFile(), TemplateMetadata.class);
    templateMetadata.setChecksum(checksum);
    long updateTime = importTemplate.getTemplateToImport().getUpdateDate().getTime();
    templateMetadata.setLastModified(updateTime);
    templateMetadata.setUpdateAvailable(false);
    mapper.writer(new DefaultPrettyPrinter()).writeValue(metadataPath.toFile(), templateMetadata);
    log.info("Generated checksum for {} - {}", templateMetadata.getTemplateName(), checksum);
    templateUpdateInfoCache.put(templateMetadata.getTemplateName(), false);
    return importTemplate;
}

Example 5 with TemplateMetadata

use of com.thinkbiganalytics.repository.api.TemplateMetadata in project kylo by Teradata.

the class RepositoryMonitor method createMetadata.

private void createMetadata(Path templateFilePath, Map<String, RegisteredTemplate> registeredTemplateMap) {
    RegisteredTemplate tmplt = null;
    try {
        File templateZipFile = templateFilePath.toFile();
        byte[] content = ImportUtil.streamToByteArray(new FileInputStream(templateZipFile));
        InputStream inputStream = new ByteArrayInputStream(content);
        ImportTemplate importTemplate = ImportUtil.openZip(templateFilePath.getFileName().toString(), inputStream);
        tmplt = importTemplate.getTemplateToImport();
        File json = getMetadataFileName(templateFilePath).toFile();
        String checksum = DigestUtils.md5DigestAsHex(content);
        boolean updateAvailable = false;
        TemplateMetadata metadata = new TemplateMetadata(tmplt.getTemplateName(), tmplt.getDescription(), templateFilePath.getFileName().toString(), checksum, tmplt.isStream(), updateAvailable, tmplt.getUpdateDate().getTime());
        // startup or new template is published
        if (!json.exists()) {
            if (registeredTemplateMap.containsKey(tmplt.getTemplateName())) {
                final String templateName = tmplt.getTemplateName();
                ExportTemplate zipFile = metadataAccess.read(() -> {
                    this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_TEMPLATES);
                    return templateExporter.exportTemplate(registeredTemplateMap.get(templateName).getId());
                }, MetadataAccess.SERVICE);
                String digest = DigestUtils.md5DigestAsHex(zipFile.getFile());
                updateAvailable = !StringUtils.equals(checksum, digest);
                metadata.setUpdateAvailable(updateAvailable);
                templateUpdateInfoCache.put(templateName, updateAvailable);
            }
        } else {
            // if registered template was updated
            metadata = mapper.readValue(json, TemplateMetadata.class);
            templateUpdateInfoCache.put(tmplt.getTemplateName(), metadata.isUpdateAvailable());
            if (metadata.getLastModified() >= tmplt.getUpdateDate().getTime()) {
                return;
            }
            if (StringUtils.equals(metadata.getChecksum(), checksum)) {
                return;
            }
            metadata.setUpdateAvailable(true);
            templateUpdateInfoCache.put(tmplt.getTemplateName(), true);
        }
        log.info("Writing template metadata for {}.", tmplt.getTemplateName());
        mapper.writerWithDefaultPrettyPrinter().writeValue(json, metadata);
    } catch (Exception e) {
        log.error("Error occurred trying to generate template metadata.", e);
        if (tmplt != null) {
            templateUpdateInfoCache.invalidate(tmplt.getTemplateName());
        }
    }
}