Examples with IOConfiguration eu.esdihumboldt.hale.common.core.io.project.model.IOConfiguration used on opensource projects

Example 11 with IOConfiguration

use of eu.esdihumboldt.hale.common.core.io.project.model.IOConfiguration in project hale by halestudio.

the class AppSchemaFileWriterTest method loadTestProject.

@BeforeClass
public static void loadTestProject() {
    try {
        URL archiveLocation = AppSchemaFileWriterTest.class.getResource(PROJECT_LOCATION);
        ArchiveProjectReader projectReader = new ArchiveProjectReader();
        projectReader.setSource(new DefaultInputSupplier(archiveLocation.toURI()));
        IOReport report = projectReader.execute(new LogProgressIndicator());
        if (!report.isSuccess()) {
            throw new RuntimeException("project reader execution failed");
        }
        tempDir = projectReader.getTemporaryFiles().iterator().next();
        project = projectReader.getProject();
        assertNotNull(project);
        sourceSchemaSpace = new DefaultSchemaSpace();
        targetSchemaSpace = new DefaultSchemaSpace();
        // load schemas
        List<IOConfiguration> resources = project.getResources();
        for (IOConfiguration resource : resources) {
            String actionId = resource.getActionId();
            String providerId = resource.getProviderId();
            // get provider
            IOProvider provider = null;
            IOProviderDescriptor descriptor = IOProviderExtension.getInstance().getFactory(providerId);
            if (descriptor == null) {
                throw new RuntimeException("Could not load I/O provider with ID: " + resource.getProviderId());
            }
            provider = descriptor.createExtensionObject();
            provider.loadConfiguration(resource.getProviderConfiguration());
            prepareProvider(provider, project, tempDir.toURI());
            IOReport providerReport = provider.execute(new LogProgressIndicator());
            if (!providerReport.isSuccess()) {
                throw new RuntimeException("I/O provider execution failed");
            }
            // TODO: could (should?) be done by an advisor
            if (provider instanceof SchemaReader) {
                Schema schema = ((SchemaReader) provider).getSchema();
                if (actionId.equals(SchemaIO.ACTION_LOAD_SOURCE_SCHEMA)) {
                    sourceSchemaSpace.addSchema(schema);
                } else if (actionId.equals(SchemaIO.ACTION_LOAD_TARGET_SCHEMA)) {
                    targetSchemaSpace.addSchema(schema);
                }
            }
        }
        // load alignment
        List<ProjectFileInfo> projectFiles = project.getProjectFiles();
        for (ProjectFileInfo projectFile : projectFiles) {
            if (projectFile.getName().equals(AlignmentIO.PROJECT_FILE_ALIGNMENT)) {
                AlignmentReader alignReader = new JaxbAlignmentReader();
                alignReader.setSource(new DefaultInputSupplier(projectFile.getLocation()));
                alignReader.setSourceSchema(sourceSchemaSpace);
                alignReader.setTargetSchema(targetSchemaSpace);
                alignReader.setPathUpdater(new PathUpdate(null, null));
                IOReport alignReport = alignReader.execute(new LogProgressIndicator());
                if (!alignReport.isSuccess()) {
                    throw new RuntimeException("alignment reader execution failed");
                }
                alignment = alignReader.getAlignment();
                assertNotNull(alignment);
                break;
            }
        }
    } catch (Exception e) {
        log.error("Exception occurred", e);
        fail("Test project could not be loaded: " + e.getMessage());
    }
}

Example 12 with IOConfiguration

use of eu.esdihumboldt.hale.common.core.io.project.model.IOConfiguration in project hale by halestudio.

the class ExamplesContent method getMappingContent.

/**
 * Get the mapping documentation content for an example project.
 *
 * @param projectId the project ID
 * @return the mapping documentation content stream or <code>null</code>
 */
private InputStream getMappingContent(String projectId) {
    if (!mappingDocExportInitialized) {
        mappingDocExport = HaleIO.createIOProvider(AlignmentWriter.class, null, ID_MAPPING_EXPORT);
        if (mappingDocExport == null) {
            log.error("Could not create mapping documentation exporter.");
        }
        mappingDocExportInitialized = true;
    }
    if (mappingDocExport == null) {
        // no mapping documentation export possible
        return null;
    }
    if (tempMappingDir == null) {
        tempMappingDir = Files.createTempDir();
        tempMappingDir.deleteOnExit();
    }
    // the file of the mapping documentation
    File mappingDoc = new File(tempMappingDir, projectId + ".html");
    if (!mappingDoc.exists()) {
        ATransaction trans = log.begin("Generate example mapping documentation");
        try {
            // create the mapping documentation
            ExampleProject exampleProject = ExampleProjectExtension.getInstance().get(projectId);
            final Project project = (Project) exampleProject.getInfo();
            // determine alignment location - contained in project file, not
            // a resource
            URI alignmentLoc = exampleProject.getAlignmentLocation();
            if (alignmentLoc == null) {
                // no alignment present
                return null;
            }
            // store configurations per action ID
            Multimap<String, IOConfiguration> confs = HashMultimap.create();
            for (IOConfiguration conf : project.getResources()) {
                confs.put(conf.getActionId(), conf);
            }
            // load schemas
            // source schemas
            LoadSchemaAdvisor source = new LoadSchemaAdvisor(SchemaSpaceID.SOURCE);
            for (IOConfiguration conf : confs.get(SchemaIO.ACTION_LOAD_SOURCE_SCHEMA)) {
                source.setConfiguration(conf);
                executeProvider(source, conf.getProviderId(), null);
            }
            // target schemas
            LoadSchemaAdvisor target = new LoadSchemaAdvisor(SchemaSpaceID.TARGET);
            for (IOConfiguration conf : confs.get(SchemaIO.ACTION_LOAD_TARGET_SCHEMA)) {
                target.setConfiguration(conf);
                executeProvider(target, conf.getProviderId(), null);
            }
            // load alignment
            // manual loading needed, as we can't rely on the environment
            // alignment advisor
            DefaultInputSupplier alignmentIn = new DefaultInputSupplier(alignmentLoc);
            AlignmentReader reader = HaleIO.findIOProvider(AlignmentReader.class, alignmentIn, alignmentLoc.getPath());
            LoadAlignmentAdvisor alignmentAdvisor = new LoadAlignmentAdvisor(null, source.getSchemaSpace(), target.getSchemaSpace(), exampleProject.getUpdater());
            reader.setSource(alignmentIn);
            executeProvider(alignmentAdvisor, null, reader);
            Alignment alignment = alignmentAdvisor.getAlignment();
            if (alignment != null) {
                // save alignment docu
                synchronized (mappingDocExport) {
                    // only a single instance
                    mappingDocExport.setAlignment(alignment);
                    mappingDocExport.setTarget(new FileIOSupplier(mappingDoc));
                    if (mappingDocExport instanceof ProjectInfoAware) {
                        ProjectInfo smallInfo = new ProjectInfo() {

                            @Override
                            public String getName() {
                                return project.getName();
                            }

                            @Override
                            public Date getModified() {
                                return null;
                            }

                            @Override
                            public Version getHaleVersion() {
                                return null;
                            }

                            @Override
                            public String getDescription() {
                                return project.getDescription();
                            }

                            @Override
                            public Date getCreated() {
                                return null;
                            }

                            @Override
                            public String getAuthor() {
                                return project.getAuthor();
                            }
                        };
                        // project);
                        ((ProjectInfoAware) mappingDocExport).setProjectInfo(smallInfo);
                    }
                    mappingDocExport.execute(null);
                }
                mappingDoc.deleteOnExit();
            }
        } catch (Throwable e) {
            log.error("Error generating mapping documentation for example project", e);
            return null;
        } finally {
            trans.end();
        }
    }
    if (mappingDoc.exists()) {
        try {
            return new FileInputStream(mappingDoc);
        } catch (FileNotFoundException e) {
            return null;
        }
    } else
        return null;
}

Example 13 with IOConfiguration

use of eu.esdihumboldt.hale.common.core.io.project.model.IOConfiguration in project hale by halestudio.

the class ProjectTest method testSaveLoad.

/**
 * Test saving and loading an example project
 *
 * @throws Exception if an error occurs
 */
@Test
public void testSaveLoad() throws Exception {
    // populate project
    Project project = new Project();
    String author;
    project.setAuthor(author = "Simon");
    String name;
    project.setName(name = "Testprojekt");
    Date created;
    project.setCreated(created = new Date(0));
    Date modified;
    project.setModified(modified = new Date());
    Version haleVersion;
    project.setHaleVersion(haleVersion = new Version("2.2.0.alpha"));
    String desc;
    project.setDescription(desc = "Hallo Welt!\nBist Du auch hier?\nÖhm.");
    IOConfiguration conf1;
    project.getResources().add(conf1 = new IOConfiguration());
    String advisorId1;
    conf1.setActionId(advisorId1 = "some advisor");
    String providerId1;
    conf1.setProviderId(providerId1 = "some provider");
    String key1;
    Value value1;
    conf1.getProviderConfiguration().put(key1 = "some key", value1 = Value.of("some value"));
    Value value2;
    String key2;
    conf1.getProviderConfiguration().put(key2 = "some other key", value2 = Value.of("some other value"));
    IOConfiguration conf2;
    project.getResources().add(conf2 = new IOConfiguration());
    String advisorId2;
    conf2.setActionId(advisorId2 = "a certain advisor");
    String providerId2;
    conf2.setProviderId(providerId2 = "a special provider");
    // write project
    File projectFile = tmp.newFile("project.xml");
    System.out.println(projectFile.getAbsolutePath());
    Project.save(project, new FileOutputStream(projectFile));
    // load project
    Project p2 = Project.load(new FileInputStream(projectFile));
    // test project
    assertEquals(author, p2.getAuthor());
    assertEquals(name, p2.getName());
    assertEquals(created, p2.getCreated());
    assertEquals(modified, p2.getModified());
    assertEquals(haleVersion, p2.getHaleVersion());
    assertEquals(desc, p2.getDescription());
    assertEquals(2, p2.getResources().size());
    Iterator<IOConfiguration> it = p2.getResources().iterator();
    IOConfiguration c1 = it.next();
    assertNotNull(c1);
    assertEquals(advisorId1, c1.getActionId());
    assertEquals(providerId1, c1.getProviderId());
    assertEquals(2, c1.getProviderConfiguration().size());
    assertTrue(c1.getProviderConfiguration().get(key1).getValue().equals(value1.getValue()));
    assertTrue(c1.getProviderConfiguration().get(key2).getValue().equals(value2.getValue()));
    IOConfiguration c2 = it.next();
    assertNotNull(c2);
    assertEquals(advisorId2, c2.getActionId());
    assertEquals(providerId2, c2.getProviderId());
}

Example 14 with IOConfiguration

use of eu.esdihumboldt.hale.common.core.io.project.model.IOConfiguration in project hale by halestudio.

the class ArchiveProjectWriter method createProjectArchive.

/**
 * Creates the project archive.
 *
 * @param target {@link OutputStream} to write the archive to
 * @param reporter the reporter to use for the execution report
 * @param progress the progress indicator
 * @return the execution report
 * @throws IOException if an I/O operation fails
 * @throws IOProviderConfigurationException if the I/O provider was not
 *             configured properly
 */
public IOReport createProjectArchive(OutputStream target, IOReporter reporter, ProgressIndicator progress) throws IOException, IOProviderConfigurationException {
    ZipOutputStream zip = new ZipOutputStream(target);
    // all files related to the project are copied into a temporary
    // directory first and then packed into a zip file
    // create temporary directory and project file
    File tempDir = Files.createTempDir();
    File baseFile = new File(tempDir, "project.halex");
    // mark the temporary directory for clean-up if the project is closed
    CleanupService clean = HalePlatform.getService(CleanupService.class);
    if (clean != null) {
        clean.addTemporaryFiles(CleanupContext.PROJECT, tempDir);
    }
    LocatableOutputSupplier<OutputStream> out = new FileIOSupplier(baseFile);
    // false is correct if getParameter is null because false is default
    boolean includeWebresources = getParameter(INCLUDE_WEB_RESOURCES).as(Boolean.class, false);
    SubtaskProgressIndicator subtask = new SubtaskProgressIndicator(progress);
    // save old IO configurations
    List<IOConfiguration> oldResources = new ArrayList<IOConfiguration>();
    for (int i = 0; i < getProject().getResources().size(); i++) {
        // clone all IO configurations to work on different objects
        oldResources.add(getProject().getResources().get(i).clone());
    }
    IOConfiguration config = getProject().getSaveConfiguration();
    if (config == null) {
        config = new IOConfiguration();
    }
    IOConfiguration oldSaveConfig = config.clone();
    // copy resources to the temp directory and update xml schemas
    updateResources(tempDir, includeWebresources, subtask, reporter);
    // update target save configuration of the project
    config.getProviderConfiguration().put(PARAM_TARGET, Value.of(baseFile.toURI().toString()));
    // write project file via XMLProjectWriter
    XMLProjectWriter writer = new XMLProjectWriter();
    writer.setTarget(out);
    writer.setProject(getProject());
    writer.setProjectFiles(getProjectFiles());
    IOReport report = writer.execute(progress, reporter);
    // now after the project with its project files is written, look for the
    // alignment file and update it
    ProjectFileInfo newAlignmentInfo = getAlignmentFile(getProject());
    if (newAlignmentInfo != null) {
        URI newAlignment = tempDir.toURI().resolve(newAlignmentInfo.getLocation());
        XMLAlignmentUpdater.update(new File(newAlignment), newAlignment, includeWebresources, reporter);
    }
    // put the complete temp directory into a zip file
    IOUtils.zipDirectory(tempDir, zip);
    zip.close();
    // the files may not be deleted now as they will be needed if the
    // project is saved again w/o loading it first
    // update the relative resource locations
    LocationUpdater updater = new LocationUpdater(getProject(), out.getLocation());
    // resources are made absolute (else they can't be found afterwards),
    // e.g. when saving the project again before loading it
    updater.updateProject(false);
    // reset the save configurations that has been overridden by the XML
    // project writer
    getProject().setSaveConfiguration(oldSaveConfig);
    if (clean == null) {
        // if no clean service is available, assume the directory is not
        // needed anymore
        FileUtils.deleteDirectory(tempDir);
    }
    return report;
}

Example 15 with IOConfiguration

use of eu.esdihumboldt.hale.common.core.io.project.model.IOConfiguration in project hale by halestudio.

the class ArchiveProjectWriter method updateResources.

/**
 * Update the resources and copy them into the target directory
 *
 * @param targetDirectory target directory
 * @param includeWebResources whether to include web resources in the copy
 * @param progress the progress indicator
 * @param reporter the reporter to use for the execution report
 * @throws IOException if an I/O operation fails
 */
protected void updateResources(File targetDirectory, boolean includeWebResources, ProgressIndicator progress, IOReporter reporter) throws IOException {
    progress.begin("Copy resources", ProgressIndicator.UNKNOWN);
    try {
        List<IOConfiguration> resources = getProject().getResources();
        // every resource needs his own directory
        int count = 1;
        // true if excluded files should be skipped; false is default
        boolean excludeDataFiles = getParameter(EXLUDE_DATA_FILES).as(Boolean.class, false);
        // resource locations mapped to new resource path
        Map<URI, String> handledResources = new HashMap<>();
        Iterator<IOConfiguration> iter = resources.iterator();
        while (iter.hasNext()) {
            IOConfiguration resource = iter.next();
            // import not possible due to cycle errors
            if (excludeDataFiles && resource.getActionId().equals("eu.esdihumboldt.hale.io.instance.read.source")) {
                // delete reference in project file
                iter.remove();
                continue;
            }
            // get resource path
            Map<String, Value> providerConfig = resource.getProviderConfiguration();
            String path = providerConfig.get(ImportProvider.PARAM_SOURCE).toString();
            URI pathUri;
            try {
                pathUri = new URI(path);
            } catch (URISyntaxException e1) {
                reporter.error(new IOMessageImpl("Skipped resource because of invalid URI: " + path, e1));
                continue;
            }
            if (!pathUri.isAbsolute()) {
                if (getPreviousTarget() != null) {
                    pathUri = getPreviousTarget().resolve(pathUri);
                } else {
                    log.warn("Could not resolve relative path " + pathUri.toString());
                }
            }
            // check if path was already handled
            if (handledResources.containsKey(pathUri)) {
                providerConfig.put(ImportProvider.PARAM_SOURCE, Value.of(handledResources.get(pathUri)));
                // skip copying the resource
                continue;
            }
            String scheme = pathUri.getScheme();
            LocatableInputSupplier<? extends InputStream> input = null;
            if (scheme != null) {
                if (scheme.equals("http") || scheme.equals("https")) {
                    // web resource
                    if (includeWebResources) {
                        input = new DefaultInputSupplier(pathUri);
                    } else {
                        // web resource that should not be included this
                        // time
                        // but the resolved URI should be stored
                        // nevertheless
                        // otherwise the URI may be invalid if it was
                        // relative
                        providerConfig.put(ImportProvider.PARAM_SOURCE, Value.of(pathUri.toASCIIString()));
                        continue;
                    }
                } else if (scheme.equals("file") || scheme.equals("platform") || scheme.equals("bundle") || scheme.equals("jar")) {
                    // files need always to be included
                    // platform resources (or other internal resources)
                    // should be included as well
                    input = new DefaultInputSupplier(pathUri);
                } else {
                    // other type of URI, e.g. JDBC
                    // not to be included
                    providerConfig.put(ImportProvider.PARAM_SOURCE, Value.of(pathUri.toASCIIString()));
                    continue;
                }
            } else {
                // now can't open that, can we?
                reporter.error(new IOMessageImpl("Skipped resource because it cannot be loaded from " + pathUri.toString(), null));
                continue;
            }
            progress.setCurrentTask("Copying resource at " + path);
            // every resource file is copied into an own resource
            // directory in the target directory
            String resourceFolder = "resource" + count;
            File newDirectory = new File(targetDirectory, resourceFolder);
            try {
                newDirectory.mkdir();
            } catch (SecurityException e) {
                throw new IOException("Can not create directory " + newDirectory.toString(), e);
            }
            // Extract the file name from pathUri.getPath().
            // This will produce a non-URL-encoded file name to be used in
            // the File(File parent, String child) constructor below
            String fileName = FilenameUtils.getName(pathUri.getPath().toString());
            if (path.isEmpty()) {
                fileName = "file";
            }
            File newFile = new File(newDirectory, fileName);
            Path target = newFile.toPath();
            // retrieve the resource advisor
            Value ct = providerConfig.get(ImportProvider.PARAM_CONTENT_TYPE);
            IContentType contentType = null;
            if (ct != null) {
                contentType = HalePlatform.getContentTypeManager().getContentType(ct.as(String.class));
            }
            ResourceAdvisor ra = ResourceAdvisorExtension.getInstance().getAdvisor(contentType);
            // copy the resource
            progress.setCurrentTask("Copying resource at " + path);
            // Extract the URL-encoded file name of the copied resource and
            // build the new relative resource path
            String resourceName = FilenameUtils.getName(target.toUri().toString());
            String newPath = resourceFolder + "/" + resourceName;
            boolean skipCopy = getParameter(EXCLUDE_CACHED_RESOURCES).as(Boolean.class, false) && !resource.getCache().isEmpty();
            if (!skipCopy) {
                ra.copyResource(input, target, contentType, includeWebResources, reporter);
                // store new path for resource
                handledResources.put(pathUri, newPath);
            }
            // update the provider configuration
            providerConfig.put(ImportProvider.PARAM_SOURCE, Value.of(newPath));
            count++;
        }
    } finally {
        progress.end();
    }
}