Examples with CqlEvaluationRequest com.ibm.cohort.cql.evaluation.CqlEvaluationRequest used on opensource projects

Example 1 with CqlEvaluationRequest

use of com.ibm.cohort.cql.evaluation.CqlEvaluationRequest in project quality-measure-and-cohort-service by Alvearie.

the class SparkCqlEvaluator method evaluate.

/**
 * Evaluate the input CQL for a single context + data pair.
 *
 * @param rowsByContext   In-memory data for all datatypes related to a single
 *                        context
 * @param contextName     Name of the context used to select measure evaluations.
 * @param resultsSchema   StructType containing the schema data for the output table
 *                        that will be created.
 * @param evaluator       configured CQLEvaluator (data provider, term provider,
 *                        library provider all previously setup)
 * @param requests        CqlEvaluationRequests containing lists of libraries,
 *                        expressions, and parameters to evaluate
 * @param columnEncoder   Encoder used to calculate output column names for
 *                        evaluation results
 * @param perContextAccum Spark accumulator that tracks each individual context
 *                        evaluation
 * @param errorAccum       Spark accumulator that tracks CQL evaluation errors
 * @param batchRunTime    Single unified timestamp for all contexts
 * @return Evaluation results for all expressions evaluated keyed by the context
 *         ID. Expression names are automatically namespaced according to the
 *         library name to avoid issues arising for expression names matching
 *         between libraries (e.g. LibraryName.ExpressionName).
 */
protected Iterator<Tuple2<Object, Row>> evaluate(Tuple2<Object, List<Row>> rowsByContext, String contextName, StructType resultsSchema, CqlEvaluator evaluator, CqlEvaluationRequests requests, SparkOutputColumnEncoder columnEncoder, LongAccumulator perContextAccum, CollectionAccumulator<EvaluationError> errorAccum, ZonedDateTime batchRunTime) {
    perContextAccum.add(1);
    List<CqlEvaluationRequest> requestsForContext = requests.getEvaluationsForContext(contextName);
    // parameters json -> {columnName, result}
    Map<String, Map<String, Object>> expressionResultsByParameters = new HashMap<>();
    for (CqlEvaluationRequest request : requestsForContext) {
        String parametersJson = encodedParametersCache.getKeyParametersColumnData(request);
        Map<String, Object> expressionResults = expressionResultsByParameters.computeIfAbsent(parametersJson, x -> new HashMap<>());
        for (CqlExpressionConfiguration expression : request.getExpressions()) {
            CqlEvaluationRequest singleRequest = new CqlEvaluationRequest(request);
            singleRequest.setExpressions(Collections.singleton(expression));
            try {
                CqlEvaluationResult result = evaluator.evaluate(singleRequest, args.debug ? CqlDebug.DEBUG : CqlDebug.NONE, batchRunTime);
                for (Map.Entry<String, Object> entry : result.getExpressionResults().entrySet()) {
                    String outputColumnKey = columnEncoder.getColumnName(request, entry.getKey());
                    expressionResults.put(outputColumnKey, typeConverter.toSparkType(entry.getValue()));
                }
            } catch (Throwable th) {
                if (args.haltOnError) {
                    throw new RuntimeException(String.format("CQL evaluation failed for ContextName: %s, OutputColumn: %s", String.valueOf(contextName), singleRequest.getExpressionNames()), th);
                } else {
                    Object contextId = rowsByContext._1();
                    errorAccum.add(new EvaluationError(contextName, contextId, singleRequest.getExpressionNames().iterator().next(), th.getMessage()));
                }
            }
        }
    }
    List<Tuple2<Object, Row>> rows = new ArrayList<>();
    for (Map.Entry<String, Map<String, Object>> entry : expressionResultsByParameters.entrySet()) {
        Object contextKey = rowsByContext._1();
        Map<String, Object> results = entry.getValue();
        Object[] data = new Object[resultsSchema.fields().length];
        data[0] = contextKey;
        data[1] = entry.getKey();
        for (int i = 2; i < resultsSchema.fieldNames().length; i++) {
            data[i] = results.get(resultsSchema.fieldNames()[i]);
        }
        rows.add(new Tuple2<Object, Row>(contextKey, RowFactory.create(data)));
    }
    return rows.iterator();
}

Example 2 with CqlEvaluationRequest

use of com.ibm.cohort.cql.evaluation.CqlEvaluationRequest in project quality-measure-and-cohort-service by Alvearie.

the class SparkCqlEvaluator method getFilteredRequests.

/**
 * @param requests     Request object to filter.
 * @param libraries    Map of library id to version used for filtering
 *                     down request based on library id. If this argument
 *                     is null or empty, then no library id filtering
 *                     is performed.
 * @param expressions  Used to optionally override which expressions will
 *                     run for each individual CqlEvaluationRequest. If this
 *                     argument is null or empty, no expressions are overwritten.
 *
 * @return CqlEvaluationRequests with the original requests optionally filtered
 *         based on the library ids the.
 *         Requests will optionally have their expressions overridden
 *         by args.expressions. if any are provided.
 *         Individual requests will also will also have any global
 *         parameters set on each individual CqlEvaluationRequest.
 */
protected CqlEvaluationRequests getFilteredRequests(CqlEvaluationRequests requests, Map<String, String> libraries, Collection<String> expressions) {
    if (requests != null) {
        List<CqlEvaluationRequest> evaluations = requests.getEvaluations();
        if (libraries != null && !libraries.isEmpty()) {
            evaluations = evaluations.stream().filter(r -> libraries.keySet().contains(r.getDescriptor().getLibraryId())).collect(Collectors.toList());
        }
        if (expressions != null && !expressions.isEmpty()) {
            evaluations.forEach(x -> x.setExpressions(x.getExpressions().stream().filter(e -> expressions.contains(e.getName())).collect(Collectors.toSet())));
        }
        if (requests.getGlobalParameters() != null) {
            for (CqlEvaluationRequest evaluation : evaluations) {
                for (Map.Entry<String, Parameter> globalParameter : requests.getGlobalParameters().entrySet()) {
                    Map<String, Parameter> parameters = evaluation.getParameters();
                    if (parameters == null) {
                        evaluation.setParameters(new HashMap<>());
                        parameters = evaluation.getParameters();
                    }
                    parameters.putIfAbsent(globalParameter.getKey(), globalParameter.getValue());
                }
            }
        }
        requests.setEvaluations(evaluations);
        jobSpecification.set(requests);
    }
    return requests;
}

Example 3 with CqlEvaluationRequest

use of com.ibm.cohort.cql.evaluation.CqlEvaluationRequest in project quality-measure-and-cohort-service by Alvearie.

the class ColumnRuleCreator method getDataRequirementsForContext.

/**
 * Retrieve the merged set of data type and column filters for all CQL jobs that will
 * be evaluated for a given aggregation context.
 *
 * @param context ContextDefinition whose CQL jobs will be interrogated for data requirements
 * @return Map of data type to the fields in that datatype that are used by the CQL jobs
 */
public Map<String, Set<StringMatcher>> getDataRequirementsForContext(ContextDefinition context) {
    Map<CqlLibraryDescriptor, Set<String>> expressionsByLibrary = new HashMap<>();
    for (CqlEvaluationRequest request : requests) {
        Set<String> expressions = expressionsByLibrary.computeIfAbsent(request.getDescriptor(), desc -> new HashSet<>());
        request.getExpressions().stream().forEach(exp -> expressions.add(exp.getName()));
    }
    DataTypeRequirementsProcessor requirementsProcessor = new DataTypeRequirementsProcessor(cqlTranslator);
    Map<String, Set<StringMatcher>> pathsByDataType = new HashMap<>();
    for (Map.Entry<CqlLibraryDescriptor, Set<String>> entry : expressionsByLibrary.entrySet()) {
        LOG.debug("Extracting data requirements for {}", entry.getKey());
        DataTypeRequirementsProcessor.DataTypeRequirements requirements = requirementsProcessor.getDataRequirements(libraryProvider, entry.getKey(), entry.getValue());
        Map<String, Set<StringMatcher>> newPaths = requirements.allAsStringMatcher();
        newPaths.forEach((key, value) -> {
            pathsByDataType.merge(key, value, (prev, current) -> {
                prev.addAll(current);
                return prev;
            });
        });
    }
    Set<StringMatcher> contextFields = pathsByDataType.computeIfAbsent(context.getPrimaryDataType(), dt -> new HashSet<>());
    contextFields.add(new EqualsStringMatcher(context.getPrimaryKeyColumn()));
    if (context.getRelationships() != null) {
        for (Join join : context.getRelationships()) {
            Set<StringMatcher> joinFields = pathsByDataType.get(join.getRelatedDataType());
            if (joinFields != null) {
                joinFields.add(new EqualsStringMatcher(join.getRelatedKeyColumn()));
                joinFields.add(new EqualsStringMatcher(ContextRetriever.JOIN_CONTEXT_VALUE_IDX));
                // if the join key is not the primary key of the primary data table, then we need to add in the alternate key
                if (join.getPrimaryDataTypeColumn() != null) {
                    contextFields.add(new EqualsStringMatcher(join.getPrimaryDataTypeColumn()));
                }
                if (join instanceof ManyToMany) {
                    ManyToMany manyToMany = (ManyToMany) join;
                    Set<StringMatcher> associationFields = pathsByDataType.computeIfAbsent(manyToMany.getAssociationDataType(), dt -> new HashSet<>());
                    associationFields.add(new EqualsStringMatcher(manyToMany.getAssociationOneKeyColumn()));
                    associationFields.add(new EqualsStringMatcher(manyToMany.getAssociationManyKeyColumn()));
                }
                if (join instanceof MultiManyToMany) {
                    ManyToMany with = ((MultiManyToMany) join).getWith();
                    while (with != null) {
                        Set<StringMatcher> relatedFields = pathsByDataType.computeIfAbsent(with.getRelatedDataType(), dt -> new HashSet<>());
                        relatedFields.add(new EqualsStringMatcher(with.getRelatedKeyColumn()));
                        relatedFields.add(new EqualsStringMatcher(ContextRetriever.JOIN_CONTEXT_VALUE_IDX));
                        with = (with instanceof MultiManyToMany) ? ((MultiManyToMany) with).getWith() : null;
                    }
                }
            }
        }
    }
    pathsByDataType.values().forEach((matcherSet -> {
        matcherSet.add(new EqualsStringMatcher(ContextRetriever.SOURCE_FACT_IDX));
    }));
    return pathsByDataType;
}

Example 4 with CqlEvaluationRequest

use of com.ibm.cohort.cql.evaluation.CqlEvaluationRequest in project quality-measure-and-cohort-service by Alvearie.

the class SparkCqlEvaluatorTest method testParameterMatrixOutputSimpleSuccess.

@Test
public void testParameterMatrixOutputSimpleSuccess() throws Exception {
    String outputLocation = "target/output/param-matrix/patient_cohort";
    CqlEvaluationRequest template = new CqlEvaluationRequest();
    template.setDescriptor(new CqlLibraryDescriptor().setLibraryId("SampleLibrary").setVersion("1.0.0"));
    template.setExpressionsByNames(Collections.singleton("IsFemale"));
    template.setContextKey("Patient");
    template.setContextValue("NA");
    CqlEvaluationRequests requests = new CqlEvaluationRequests();
    requests.setEvaluations(new ArrayList<>());
    List<Integer> ages = Arrays.asList(15, 17, 18);
    for (Integer age : ages) {
        Map<String, Parameter> parameters = new HashMap<>();
        parameters.put("MinimumAge", new IntegerParameter(age));
        CqlEvaluationRequest request = new CqlEvaluationRequest(template);
        request.setParameters(parameters);
        requests.getEvaluations().add(request);
    }
    ObjectMapper om = new ObjectMapper();
    File jobsFile = new File("target/output/param-matrix-simple/cql-jobs.json");
    if (!jobsFile.exists()) {
        jobsFile.getParentFile().mkdirs();
    }
    FileUtils.write(jobsFile, om.writeValueAsString(requests), StandardCharsets.UTF_8);
    try {
        String[] args = new String[] { "-d", "src/test/resources/simple-job/context-definitions.json", "-j", jobsFile.getPath(), "-m", "src/test/resources/simple-job/modelinfo/simple-modelinfo-1.0.0.xml", "-c", "src/test/resources/simple-job/cql", "-i", "Patient=" + new File("src/test/resources/simple-job/testdata/patient").toURI().toString(), "-o", "Patient=" + new File(outputLocation).toURI().toString(), "--output-format", "delta", "--overwrite-output-for-contexts", "--metadata-output-path", outputLocation };
        SparkCqlEvaluator.main(args);
        validateOutputCountsAndColumns(outputLocation, new HashSet<>(Arrays.asList("id", "parameters", "SampleLibrary|IsFemale")), 10 * ages.size(), "delta");
    } finally {
        jobsFile.delete();
    }
}

Example 5 with CqlEvaluationRequest

use of com.ibm.cohort.cql.evaluation.CqlEvaluationRequest in project quality-measure-and-cohort-service by Alvearie.

the class SparkCqlEvaluatorTest method testGetFilteredRequestsFilterToLibrariesIgnoresVersion.

@Test
public void testGetFilteredRequestsFilterToLibrariesIgnoresVersion() {
    CqlEvaluationRequests requests = new CqlEvaluationRequests();
    CqlEvaluationRequest request = makeEvaluationRequest("context", "lib1", "1.0.0");
    request.setExpressionsByNames(new HashSet<>(Collections.singletonList("cohort")));
    CqlEvaluationRequest request2 = makeEvaluationRequest("context", "lib2", "1.0.0");
    request2.setExpressionsByNames(new HashSet<>(Collections.singletonList("cohort")));
    CqlEvaluationRequest request3 = makeEvaluationRequest("context", "lib3", "1.0.0");
    request.setExpressionsByNames(new HashSet<>(Collections.singletonList("cohort")));
    CqlEvaluationRequest request4 = makeEvaluationRequest("context", "lib4", "1.0.0");
    request2.setExpressionsByNames(new HashSet<>(Collections.singletonList("cohort")));
    List<CqlEvaluationRequest> evaluations = Arrays.asList(request, request2, request3, request4);
    requests.setEvaluations(evaluations);
    Map<String, String> libs = new HashMap<String, String>() {

        {
            put("lib3", "7.0.0");
            put("lib4", "1.0.0");
        }
    };
    CqlEvaluationRequests actual = evaluator.getFilteredRequests(requests, libs, null);
    assertEquals(2, actual.getEvaluations().size());
    for (CqlEvaluationRequest cqlEvaluationRequest : actual.getEvaluations()) {
        assertTrue(libs.containsKey(cqlEvaluationRequest.getDescriptor().getLibraryId()));
    }
}