Examples with JoinParameter eu.esdihumboldt.hale.common.align.model.functions.join.JoinParameter used on opensource projects

Example 11 with JoinParameter

use of eu.esdihumboldt.hale.common.align.model.functions.join.JoinParameter in project hale by halestudio.

the class JoinHandler method handleTypeTransformation.

/**
 * @see eu.esdihumboldt.hale.io.appschema.writer.internal.TypeTransformationHandler#handleTypeTransformation(eu.esdihumboldt.hale.common.align.model.Cell,
 *      eu.esdihumboldt.hale.io.appschema.writer.internal.mapping.AppSchemaMappingContext)
 */
@Override
public FeatureTypeMapping handleTypeTransformation(Cell typeCell, AppSchemaMappingContext context) {
    AppSchemaMappingWrapper mapping = context.getMappingWrapper();
    Alignment alignment = context.getAlignment();
    FeatureChaining featureChaining = context.getFeatureChaining();
    final JoinParameter joinParameter = typeCell.getTransformationParameters().get(PARAMETER_JOIN).get(0).as(JoinParameter.class);
    String validation = joinParameter.validate();
    if (validation != null)
        throw new IllegalArgumentException("Join parameter invalid: " + validation);
    // check only single predicate conditions have been used
    int[] conditionCount = new int[joinParameter.getTypes().size()];
    List<JoinCondition> joinConditions = getSortedJoinConditions(joinParameter);
    for (JoinCondition joinCondition : joinConditions) {
        TypeEntityDefinition joinType = AlignmentUtil.getTypeEntity(joinCondition.joinProperty);
        int typeIdx = joinParameter.getTypes().indexOf(joinType);
        conditionCount[typeIdx]++;
        if (conditionCount[typeIdx] > 1) {
            throw new IllegalArgumentException("Only single condition joins are supported so far");
        }
    }
    FeatureTypeMapping topMostMapping = null;
    for (int chainIdx = 0; chainIdx < joinParameter.getTypes().size() - 1; chainIdx++) {
        ChainConfiguration previousChainConf = null;
        ChainConfiguration chainConf = null;
        if (featureChaining != null) {
            chainConf = featureChaining.getChain(typeCell.getId(), chainIdx);
            if (chainConf != null && chainConf.getPrevChainIndex() >= 0) {
                previousChainConf = featureChaining.getChain(typeCell.getId(), chainConf.getPrevChainIndex());
            }
        }
        // join is done pair-wise: I assume the first type is the container
        // type, whereas the second type is nested in the first
        JoinCondition joinCondition = joinConditions.get(chainIdx);
        baseProperty = joinCondition.baseProperty;
        joinProperty = joinCondition.joinProperty;
        TypeEntityDefinition containerType = AlignmentUtil.getTypeEntity(baseProperty);
        TypeEntityDefinition nestedType = AlignmentUtil.getTypeEntity(joinProperty);
        // build FeatureTypeMapping for container type
        // Entity containerTypeTarget = typeCell.getTarget().values().iterator().next();
        // TypeDefinition containerTypeTargetType = containerTypeTarget.getDefinition().getType();
        EntityDefinition containerTypeTarget = null;
        TypeDefinition containerTypeTargetType = null;
        String containerTypeTargetMappingName = null;
        if (previousChainConf == null) {
            containerTypeTarget = getTargetType(typeCell).getDefinition();
            containerTypeTargetType = containerTypeTarget.getType();
        } else {
            containerTypeTarget = previousChainConf.getNestedTypeTarget();
            containerTypeTargetType = previousChainConf.getNestedTypeTarget().getDefinition().getPropertyType();
            containerTypeTargetMappingName = previousChainConf.getMappingName();
        }
        String containerMappingName = null;
        if (previousChainConf != null) {
            containerMappingName = previousChainConf.getMappingName();
        }
        FeatureTypeMapping containerFTMapping = context.getOrCreateFeatureTypeMapping(containerTypeTargetType, containerMappingName);
        containerFTMapping.setSourceType(containerType.getDefinition().getName().getLocalPart());
        // build FeatureTypeMapping for nested type
        TypeDefinition nestedFT = null;
        List<ChildContext> nestedFTPath = null;
        FeatureTypeMapping nestedFTMapping = null;
        if (chainConf != null) {
            nestedFT = chainConf.getNestedTypeTarget().getDefinition().getPropertyType();
            nestedFTPath = chainConf.getNestedTypeTarget().getPropertyPath();
            // remove last element
            nestedFTPath = nestedFTPath.subList(0, nestedFTPath.size() - 1);
            nestedFTMapping = context.getOrCreateFeatureTypeMapping(nestedFT, chainConf.getMappingName());
            nestedFTMapping.setSourceType(nestedType.getDefinition().getName().getLocalPart());
        } else {
            if (joinParameter.getTypes().size() > 2) {
                throw new IllegalArgumentException("If no feature chaining configuration is provided, only join between 2 types is supported");
            }
            // do your best to figure it out on your own... good luck!
            Collection<? extends Cell> propertyCells = alignment.getPropertyCells(typeCell);
            for (Cell propertyCell : propertyCells) {
                Property sourceProperty = AppSchemaMappingUtils.getSourceProperty(propertyCell);
                if (sourceProperty != null) {
                    TypeDefinition sourceType = sourceProperty.getDefinition().getDefinition().getParentType();
                    if (sourceType.getName().equals(nestedType.getDefinition().getName())) {
                        // source property belongs to nested type: determine
                        // target type
                        Property targetProperty = getTargetProperty(propertyCell);
                        // nestedFT =
                        // findOwningFeatureType(targetProperty.getDefinition());
                        nestedFT = findOwningType(targetProperty.getDefinition(), context.getRelevantTargetTypes());
                        if (nestedFT != null && !nestedFT.getName().equals(containerTypeTargetType.getName())) {
                            // target property belongs to a feature type
                            // different from the already mapped one: build
                            // a new mapping
                            nestedFTPath = findOwningTypePath(targetProperty.getDefinition(), context.getRelevantTargetTypes());
                            nestedFTMapping = context.getOrCreateFeatureTypeMapping(nestedFT);
                            nestedFTMapping.setSourceType(nestedType.getDefinition().getName().getLocalPart());
                            // in the join
                            break;
                        } else if (isHRefAttribute(targetProperty.getDefinition().getDefinition())) {
                            // check if target property is a href attribute
                            Property hrefProperty = targetProperty;
                            List<ChildContext> hrefPropertyPath = hrefProperty.getDefinition().getPropertyPath();
                            List<ChildContext> hrefContainerPath = hrefPropertyPath.subList(0, hrefPropertyPath.size() - 1);
                            TypeDefinition hrefParentType = hrefProperty.getDefinition().getDefinition().getParentType();
                            // TypeDefinition childFT =
                            // findChildFeatureType(hrefParentType);
                            TypeDefinition childFT = AppSchemaMappingUtils.findChildType(hrefParentType, context.getRelevantTargetTypes());
                            if (childFT != null) {
                                nestedFTPath = hrefContainerPath;
                                nestedFTMapping = context.getOrCreateFeatureTypeMapping(childFT);
                                nestedFTMapping.setSourceType(nestedType.getDefinition().getName().getLocalPart());
                                // involved in the join
                                break;
                            }
                        }
                    }
                }
            }
        }
        // build join mapping
        if (nestedFTMapping != null && nestedFTPath != null) {
            AttributeMappingType containerJoinMapping = context.getOrCreateAttributeMapping(containerTypeTargetType, containerTypeTargetMappingName, nestedFTPath);
            containerJoinMapping.setTargetAttribute(mapping.buildAttributeXPath(containerTypeTargetType, nestedFTPath));
            // set isMultiple attribute
            PropertyDefinition targetPropertyDef = nestedFTPath.get(nestedFTPath.size() - 1).getChild().asProperty();
            if (AppSchemaMappingUtils.isMultiple(targetPropertyDef)) {
                containerJoinMapping.setIsMultiple(true);
            }
            AttributeExpressionMappingType containerSourceExpr = new AttributeExpressionMappingType();
            // join column extracted from join condition
            containerSourceExpr.setOCQL(baseProperty.getDefinition().getName().getLocalPart());
            containerSourceExpr.setLinkElement(getLinkElementValue(nestedFTMapping));
            String linkField = context.getUniqueFeatureLinkAttribute(nestedFT, nestedFTMapping.getMappingName());
            containerSourceExpr.setLinkField(linkField);
            containerJoinMapping.setSourceExpression(containerSourceExpr);
            AttributeMappingType nestedJoinMapping = new AttributeMappingType();
            AttributeExpressionMappingType nestedSourceExpr = new AttributeExpressionMappingType();
            // join column extracted from join condition
            nestedSourceExpr.setOCQL(joinProperty.getDefinition().getName().getLocalPart());
            nestedJoinMapping.setSourceExpression(nestedSourceExpr);
            nestedJoinMapping.setTargetAttribute(linkField);
            nestedFTMapping.getAttributeMappings().getAttributeMapping().add(nestedJoinMapping);
        }
        if (chainIdx == 0) {
            topMostMapping = containerFTMapping;
        }
    }
    return topMostMapping;
}

Example 12 with JoinParameter

use of eu.esdihumboldt.hale.common.align.model.functions.join.JoinParameter in project hale by halestudio.

the class IndexJoinHandler method partitionInstances.

/**
 * @see eu.esdihumboldt.hale.common.align.transformation.function.InstanceHandler#partitionInstances(eu.esdihumboldt.hale.common.instance.model.InstanceCollection,
 *      java.lang.String,
 *      eu.esdihumboldt.hale.common.align.transformation.engine.TransformationEngine,
 *      com.google.common.collect.ListMultimap, java.util.Map,
 *      eu.esdihumboldt.hale.common.align.transformation.report.TransformationLog)
 */
@Override
public ResourceIterator<FamilyInstance> partitionInstances(InstanceCollection instances, String transformationIdentifier, TransformationEngine engine, ListMultimap<String, ParameterValue> transformationParameters, Map<String, String> executionParameters, TransformationLog log) throws TransformationException {
    if (transformationParameters == null || !transformationParameters.containsKey(PARAMETER_JOIN) || transformationParameters.get(PARAMETER_JOIN).isEmpty()) {
        throw new TransformationException("No join parameter defined");
    }
    JoinHandler fallbackHandler = new JoinHandler();
    InstanceIndexService indexService = serviceProvider.getService(InstanceIndexService.class);
    if (indexService == null) {
        log.warn(MessageFormat.format("Index service not available, falling back to join handler {0}", fallbackHandler.getClass().getCanonicalName()));
        return fallbackHandler.partitionInstances(instances, transformationIdentifier, engine, transformationParameters, executionParameters, log);
    }
    JoinParameter joinParameter = transformationParameters.get(PARAMETER_JOIN).get(0).as(JoinParameter.class);
    String validation = joinParameter.validate();
    if (validation != null) {
        throw new TransformationException("Join parameter invalid: " + validation);
    }
    List<TypeEntityDefinition> types = joinParameter.getTypes();
    JoinDefinition joinDefinition = JoinUtil.getJoinDefinition(joinParameter);
    // remember instances of first type to start join afterwards
    Collection<ResolvableInstanceReference> startInstances = new LinkedList<ResolvableInstanceReference>();
    List<Object> inputInstanceIds = new ArrayList<>();
    try (ResourceIterator<Instance> it = instances.iterator()) {
        while (it.hasNext()) {
            Instance i = InstanceDecorator.getRoot(it.next());
            // remember instances of first type
            if (i.getDefinition().equals(types.get(0).getDefinition())) {
                startInstances.add(new ResolvableInstanceReference(instances.getReference(i), instances));
            }
            if (!Identifiable.is(i)) {
                log.warn(MessageFormat.format("At least one instance does not have an ID, falling back to join handler {0}", fallbackHandler.getClass().getCanonicalName()));
                return fallbackHandler.partitionInstances(instances, transformationIdentifier, engine, transformationParameters, executionParameters, log);
            }
            inputInstanceIds.add(Identifiable.getId(i));
        }
    }
    return new IndexJoinIterator(startInstances, joinDefinition, indexService);
}

Example 13 with JoinParameter

use of eu.esdihumboldt.hale.common.align.model.functions.join.JoinParameter in project hale by halestudio.

the class JoinHandler method partitionInstances.

// For now no support for using the same type more than once in a join.
/**
 * @see eu.esdihumboldt.hale.common.align.transformation.function.InstanceHandler#partitionInstances(eu.esdihumboldt.hale.common.instance.model.InstanceCollection,
 *      java.lang.String,
 *      eu.esdihumboldt.hale.common.align.transformation.engine.TransformationEngine,
 *      com.google.common.collect.ListMultimap, java.util.Map,
 *      eu.esdihumboldt.hale.common.align.transformation.report.TransformationLog)
 */
@Override
public ResourceIterator<FamilyInstance> partitionInstances(InstanceCollection instances, String transformationIdentifier, TransformationEngine engine, ListMultimap<String, ParameterValue> transformationParameters, Map<String, String> executionParameters, TransformationLog log) throws TransformationException {
    if (transformationParameters == null || !transformationParameters.containsKey(PARAMETER_JOIN) || transformationParameters.get(PARAMETER_JOIN).isEmpty()) {
        throw new TransformationException("No join parameter defined");
    }
    JoinParameter joinParameter = transformationParameters.get(PARAMETER_JOIN).get(0).as(JoinParameter.class);
    String validation = joinParameter.validate();
    if (validation != null)
        throw new TransformationException("Join parameter invalid: " + validation);
    List<TypeEntityDefinition> types = joinParameter.getTypes();
    JoinDefinition joinDefinition = JoinUtil.getJoinDefinition(joinParameter);
    // JoinProperty -> (Value -> Collection<Reference>)
    Map<PropertyEntityDefinition, Multimap<Object, InstanceReference>> index = new HashMap<>();
    for (PropertyEntityDefinition property : joinDefinition.properties.values()) index.put(property, ArrayListMultimap.<Object, InstanceReference>create());
    // remember instances of first type to start join afterwards
    Collection<InstanceReference> startInstances = new LinkedList<InstanceReference>();
    // iterate once over all instances
    ResourceIterator<Instance> iterator = instances.iterator();
    try {
        while (iterator.hasNext()) {
            Instance next = iterator.next();
            // remember instances of first type
            if (next.getDefinition().equals(types.get(0).getDefinition())) {
                startInstances.add(instances.getReference(next));
            }
            // fill index over needed properties
            for (PropertyEntityDefinition property : joinDefinition.properties.get(next.getDefinition())) {
                // XXX what about null? for now ignore null values
                // XXX how to treat multiple values? must all be equal (in
                // order?) or only one?
                Collection<Object> values = AlignmentUtil.getValues(next, property, true);
                if (values != null && !values.isEmpty()) {
                    // XXX take only first value for now
                    index.get(property).put(valueProcessor.processValue(values.iterator().next(), property), instances.getReference(next));
                }
            }
        }
    } finally {
        iterator.close();
    }
    return new JoinIterator(instances, startInstances, joinDefinition.directParent, index, joinDefinition.joinTable, valueProcessor);
}

Example 14 with JoinParameter

use of eu.esdihumboldt.hale.common.align.model.functions.join.JoinParameter in project hale by halestudio.

the class JoinExplanation method getExplanation.

@Override
protected String getExplanation(Cell cell, boolean html, ServiceProvider services, Locale locale) {
    JoinParameter join = CellUtil.getFirstParameter(cell, PARAMETER_JOIN).as(JoinParameter.class);
    if (join != null && join.getTypes() != null && !join.getTypes().isEmpty()) {
        // types
        StringBuilder types = new StringBuilder();
        boolean first = true;
        for (TypeEntityDefinition type : join.getTypes()) {
            if (first) {
                first = false;
            } else {
                types.append(", ");
            }
            types.append(formatEntity(type, html, false, locale));
        }
        StringBuilder sb = new StringBuilder();
        sb.append(MessageFormat.format(getMessage("main", locale), types.toString()));
        sb.append("\n\n");
        for (JoinCondition condition : join.getConditions()) {
            sb.append(formatFullEntity(condition.baseProperty, html, locale));
            sb.append(" = ");
            sb.append(formatFullEntity(condition.joinProperty, html, locale));
            sb.append('\n');
        }
        sb.append('\n');
        // finalize
        String explanation = sb.toString();
        if (html)
            explanation = explanation.replaceAll("\n", "<br />");
        return explanation;
    }
    return null;
}

Example 15 with JoinParameter

use of eu.esdihumboldt.hale.common.align.model.functions.join.JoinParameter in project hale by halestudio.

the class JoinContext method apply.

/**
 * Apply information collected in the context to the cell.
 *
 * @param newCell the merged cell
 * @param log the cell log
 * @param migration the alignment migration
 */
public void apply(MutableCell newCell, AlignmentMigration migration, SimpleLog log) {
    ListMultimap<String, ParameterValue> params = ArrayListMultimap.create();
    /*
		 * Order: Keep original order but replace entities w/ all matches
		 */
    Set<TypeEntityDefinition> types = new LinkedHashSet<>();
    for (TypeEntityDefinition type : orgParameter.getTypes()) {
        List<TypeEntityDefinition> repl = replacements.get(type);
        if (repl.isEmpty()) {
            log.error("Could not find replacement for type {0} in join order", type);
            types.add(type);
        } else {
            types.addAll(repl);
        }
    }
    /*
		 * Conditions: (1) add conditions from matches and (2) add conditions
		 * from original cell translated to new schema (via property mapping),
		 * if they are not duplicates
		 */
    Set<Pair<PropertyEntityDefinition, PropertyEntityDefinition>> cons = new LinkedHashSet<>();
    // add conditions from matches
    for (Cell match : joinMatches) {
        JoinParameter matchParameter = CellUtil.getFirstParameter(match, JoinFunction.PARAMETER_JOIN).as(JoinParameter.class);
        for (JoinCondition condition : matchParameter.getConditions()) {
            cons.add(new Pair<>(condition.baseProperty, condition.joinProperty));
        }
    }
    // migrate original conditions
    Set<JoinCondition> migrated = orgParameter.getConditions().stream().map(condition -> {
        PropertyEntityDefinition baseProperty = processOriginalConditionProperty(condition.baseProperty, migration, log);
        PropertyEntityDefinition joinProperty = processOriginalConditionProperty(condition.joinProperty, migration, log);
        JoinCondition result = new JoinCondition(baseProperty, joinProperty);
        return result;
    }).collect(Collectors.toSet());
    for (JoinCondition condition : migrated) {
        if (!condition.baseProperty.equals(condition.joinProperty)) {
            // migrated condition may contain "loop" condition
            cons.add(new Pair<>(condition.baseProperty, condition.joinProperty));
        }
    }
    // add messages on dropped filter/conditions
    for (EntityDefinition stripped : strippedSources) {
        if (!AlignmentUtil.isDefaultEntity(stripped)) {
            String msg = "Conditions/contexts for an original source could not be transfered and were dropped: " + MergeUtil.getContextInfoString(stripped);
            log.warn(msg);
        }
    }
    // all conditions
    Set<JoinCondition> conditions = new HashSet<>();
    for (Pair<PropertyEntityDefinition, PropertyEntityDefinition> condition : cons) {
        conditions.add(new JoinCondition(condition.getFirst(), condition.getSecond()));
    }
    JoinParameter newParam = new JoinParameter(new ArrayList<>(types), conditions);
    params.replaceValues(JoinFunction.PARAMETER_JOIN, Collections.singleton(new ParameterValue(Value.of(newParam))));
    // Use Groovy Join if original or match uses a script
    if (!scripts.isEmpty()) {
        boolean originalScript = scripts.size() == 1 && GroovyJoin.ID.equals(newCell.getTransformationIdentifier());
        Text script;
        if (originalScript) {
            // use original script
            script = new Text(scripts.get(0).getSecond());
            // create annotation
            log.warn("The Groovy script from the original cell was reused, logic and references to sources are very likely not valid anymore.");
        } else {
            // dummy script with all original scripts
            newCell.setTransformationIdentifier(GroovyJoin.ID);
            script = buildScript(scripts);
            // create annotation
            log.warn("At least one source mapping used a Groovy script, the script could not be combined automatically and was replaced with a dummy script (old scripts are commented out). Please check how you can migrate the old functionality.");
        }
        params.replaceValues(GroovyConstants.PARAMETER_SCRIPT, Collections.singleton(new ParameterValue(Value.of(script))));
    }
    newCell.setTransformationParameters(params);
}