Search in sources :

Example 1 with ContextDefinition

use of com.ibm.cohort.cql.spark.aggregation.ContextDefinition in project quality-measure-and-cohort-service by Alvearie.

the class SparkCqlEvaluator method createDataTypeAliases.

private Map<String, String> createDataTypeAliases(List<ContextDefinition> filteredContexts, CqlToElmTranslator translator) {
    Set<String> dataTypes = filteredContexts.stream().map(ContextDefinition::getPrimaryDataType).collect(Collectors.toSet());
    Collection<ModelInfo> modelInfos = translator.getRegisteredModelInfos().values();
    Map<String, String> dataTypeAliases = new HashMap<>();
    for (ModelInfo modelInfo : modelInfos) {
        modelInfo.getTypeInfo().stream().filter(ClassInfo.class::isInstance).map(ClassInfo.class::cast).filter(classInfo -> dataTypes.contains(classInfo.getName())).forEach(info -> {
            String dataType = info.getName();
            QName baseType = ModelUtils.getBaseTypeName(modelInfo, info);
            if (baseType != null) {
                // for inheritance types
                dataTypeAliases.put(dataType, baseType.getLocalPart());
            }
            Collection<String> choiceTypes = ModelUtils.getChoiceTypeNames(info);
            // for choice types
            choiceTypes.forEach(choiceType -> dataTypeAliases.put(dataType, choiceType));
        });
    }
    return dataTypeAliases;
}
Also used : ModelInfo(org.hl7.elm_modelinfo.r1.ModelInfo) Arrays(java.util.Arrays) CqlDataProvider(com.ibm.cohort.cql.data.CqlDataProvider) CqlToElmTranslator(com.ibm.cohort.cql.translation.CqlToElmTranslator) FileSystem(org.apache.hadoop.fs.FileSystem) ZonedDateTime(java.time.ZonedDateTime) LoggerFactory(org.slf4j.LoggerFactory) EncodedParametersCache(com.ibm.cohort.cql.spark.util.EncodedParametersCache) DataRowDataProvider(com.ibm.cohort.datarow.engine.DataRowDataProvider) Format(com.ibm.cohort.cql.library.Format) ClassInfo(org.hl7.elm_modelinfo.r1.ClassInfo) ColumnRuleCreator(com.ibm.cohort.cql.spark.aggregation.ColumnRuleCreator) ConfigurableOutputColumnNameEncoder(com.ibm.cohort.cql.spark.data.ConfigurableOutputColumnNameEncoder) DefaultDatasetRetriever(com.ibm.cohort.cql.spark.data.DefaultDatasetRetriever) SparkSchemaCreator(com.ibm.cohort.cql.spark.data.SparkSchemaCreator) ContextDefinition(com.ibm.cohort.cql.spark.aggregation.ContextDefinition) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) CqlEvaluationRequest(com.ibm.cohort.cql.evaluation.CqlEvaluationRequest) ModelUtils(com.ibm.cohort.cql.spark.optimizer.ModelUtils) ConstraintViolation(javax.validation.ConstraintViolation) StructType(org.apache.spark.sql.types.StructType) HadoopPathOutputMetadataWriter(com.ibm.cohort.cql.spark.metadata.HadoopPathOutputMetadataWriter) Collection(java.util.Collection) Set(java.util.Set) Validator(javax.validation.Validator) CqlLibraryProvider(com.ibm.cohort.cql.library.CqlLibraryProvider) Reader(java.io.Reader) Tuple2(scala.Tuple2) Collectors(java.util.stream.Collectors) FileNotFoundException(java.io.FileNotFoundException) Serializable(java.io.Serializable) ContextRetriever(com.ibm.cohort.cql.spark.aggregation.ContextRetriever) List(java.util.List) CqlEvaluationRequests(com.ibm.cohort.cql.evaluation.CqlEvaluationRequests) CqlTerminologyProvider(com.ibm.cohort.cql.terminology.CqlTerminologyProvider) QName(javax.xml.namespace.QName) SparkOutputColumnEncoder(com.ibm.cohort.cql.spark.data.SparkOutputColumnEncoder) Parameter(com.ibm.cohort.cql.evaluation.parameters.Parameter) MapUtils(com.ibm.cohort.cql.util.MapUtils) ExternalFunctionProvider(org.opencds.cqf.cql.engine.data.ExternalFunctionProvider) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Dataset(org.apache.spark.sql.Dataset) SerializableConfiguration(org.apache.spark.util.SerializableConfiguration) CqlEvaluationResult(com.ibm.cohort.cql.evaluation.CqlEvaluationResult) SparkDataRow(com.ibm.cohort.cql.spark.data.SparkDataRow) PriorityCqlLibraryProvider(com.ibm.cohort.cql.library.PriorityCqlLibraryProvider) R4FileSystemFhirTerminologyProvider(com.ibm.cohort.cql.terminology.R4FileSystemFhirTerminologyProvider) HashMap(java.util.HashMap) ValidatorFactory(javax.validation.ValidatorFactory) HadoopBasedCqlLibraryProvider(com.ibm.cohort.cql.library.HadoopBasedCqlLibraryProvider) SparkTypeConverter(com.ibm.cohort.cql.spark.data.SparkTypeConverter) Function(java.util.function.Function) ArrayList(java.util.ArrayList) CustomMetricSparkPlugin(com.ibm.cohort.cql.spark.metrics.CustomMetricSparkPlugin) CollectionUtils(org.apache.commons.collections.CollectionUtils) EvaluationError(com.ibm.cohort.cql.spark.errors.EvaluationError) DataRowRetrieveProvider(com.ibm.cohort.datarow.engine.DataRowRetrieveProvider) Validation(javax.validation.Validation) TranslatingCqlLibraryProvider(com.ibm.cohort.cql.translation.TranslatingCqlLibraryProvider) NoSuchElementException(java.util.NoSuchElementException) EvaluationSummary(com.ibm.cohort.cql.spark.metadata.EvaluationSummary) DataRow(com.ibm.cohort.datarow.model.DataRow) SparkSession(org.apache.spark.sql.SparkSession) PrintStream(java.io.PrintStream) CqlEvaluator(com.ibm.cohort.cql.evaluation.CqlEvaluator) ClasspathCqlLibraryProvider(com.ibm.cohort.cql.library.ClasspathCqlLibraryProvider) SaveMode(org.apache.spark.sql.SaveMode) Logger(org.slf4j.Logger) CqlDebug(com.ibm.cohort.cql.evaluation.CqlDebug) Iterator(java.util.Iterator) RowFactory(org.apache.spark.sql.RowFactory) JCommander(com.beust.jcommander.JCommander) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) IOException(java.io.IOException) Row(org.apache.spark.sql.Row) CollectionAccumulator(org.apache.spark.util.CollectionAccumulator) InputStreamReader(java.io.InputStreamReader) JavaPairRDD(org.apache.spark.api.java.JavaPairRDD) LongAccumulator(org.apache.spark.util.LongAccumulator) CqlExpressionConfiguration(com.ibm.cohort.cql.evaluation.CqlExpressionConfiguration) AnyColumnFunctions(com.ibm.cohort.cql.functions.AnyColumnFunctions) MDC(org.slf4j.MDC) UnsupportedTerminologyProvider(com.ibm.cohort.cql.terminology.UnsupportedTerminologyProvider) CohortExternalFunctionProvider(com.ibm.cohort.cql.functions.CohortExternalFunctionProvider) ContextDefinitions(com.ibm.cohort.cql.spark.aggregation.ContextDefinitions) Collections(java.util.Collections) OutputMetadataWriter(com.ibm.cohort.cql.spark.metadata.OutputMetadataWriter) ModelInfo(org.hl7.elm_modelinfo.r1.ModelInfo) HashMap(java.util.HashMap) QName(javax.xml.namespace.QName) ClassInfo(org.hl7.elm_modelinfo.r1.ClassInfo)

Example 2 with ContextDefinition

use of com.ibm.cohort.cql.spark.aggregation.ContextDefinition in project quality-measure-and-cohort-service by Alvearie.

the class SparkSchemaCreatorTest method makeContextDefinition.

private ContextDefinition makeContextDefinition(String name, String primaryDataType, String primaryKeyColumn) {
    ContextDefinition contextDefinition = new ContextDefinition();
    contextDefinition.setName(name);
    contextDefinition.setPrimaryDataType(primaryDataType);
    contextDefinition.setPrimaryKeyColumn(primaryKeyColumn);
    return contextDefinition;
}
Also used : ContextDefinition(com.ibm.cohort.cql.spark.aggregation.ContextDefinition)

Example 3 with ContextDefinition

use of com.ibm.cohort.cql.spark.aggregation.ContextDefinition in project quality-measure-and-cohort-service by Alvearie.

the class SparkCqlEvaluator method run.

public void run(PrintStream out) throws Exception {
    EvaluationSummary evaluationSummary = new EvaluationSummary();
    long startTimeMillis = System.currentTimeMillis();
    evaluationSummary.setStartTimeMillis(startTimeMillis);
    evaluationSummary.setJobStatus(JobStatus.FAIL);
    SparkSession.Builder sparkBuilder = SparkSession.builder();
    try (SparkSession spark = sparkBuilder.getOrCreate()) {
        final LongAccumulator contextAccum = spark.sparkContext().longAccumulator("Context");
        final CollectionAccumulator<EvaluationError> errorAccumulator = spark.sparkContext().collectionAccumulator("EvaluationErrors");
        try {
            spark.sparkContext().setLocalProperty("mdc." + CORRELATION_ID, MDC.get(CORRELATION_ID));
            evaluationSummary.setCorrelationId(MDC.get(CORRELATION_ID));
            boolean useJava8API = Boolean.valueOf(spark.conf().get("spark.sql.datetime.java8API.enabled"));
            this.typeConverter = new SparkTypeConverter(useJava8API);
            this.hadoopConfiguration = new SerializableConfiguration(spark.sparkContext().hadoopConfiguration());
            evaluationSummary.setApplicationId(spark.sparkContext().applicationId());
            CqlToElmTranslator cqlTranslator = getCqlTranslator();
            SparkOutputColumnEncoder columnEncoder = getSparkOutputColumnEncoder();
            ContextDefinitions contexts = readContextDefinitions(args.contextDefinitionPath);
            List<ContextDefinition> filteredContexts = contexts.getContextDefinitions();
            if (args.aggregationContexts != null && !args.aggregationContexts.isEmpty()) {
                filteredContexts = filteredContexts.stream().filter(def -> args.aggregationContexts.contains(def.getName())).collect(Collectors.toList());
            }
            if (filteredContexts.isEmpty()) {
                throw new IllegalArgumentException("At least one context definition is required (after filtering if enabled).");
            }
            Map<String, StructType> resultSchemas = calculateSparkSchema(filteredContexts.stream().map(ContextDefinition::getName).collect(Collectors.toList()), contexts, columnEncoder, cqlTranslator);
            ZonedDateTime batchRunTime = ZonedDateTime.now();
            final LongAccumulator perContextAccum = spark.sparkContext().longAccumulator("PerContext");
            CustomMetricSparkPlugin.contextAccumGauge.setAccumulator(contextAccum);
            CustomMetricSparkPlugin.perContextAccumGauge.setAccumulator(perContextAccum);
            CustomMetricSparkPlugin.totalContextsToProcessCounter.inc(filteredContexts.size());
            CustomMetricSparkPlugin.currentlyEvaluatingContextGauge.setValue(0);
            ColumnRuleCreator columnRuleCreator = new ColumnRuleCreator(getFilteredJobSpecificationWithIds().getEvaluations(), getCqlTranslator(), createLibraryProvider());
            Map<String, String> dataTypeAliases = createDataTypeAliases(filteredContexts, cqlTranslator);
            for (ContextDefinition context : filteredContexts) {
                final String contextName = context.getName();
                ContextRetriever contextRetriever = new ContextRetriever(args.inputPaths, new DefaultDatasetRetriever(spark, args.inputFormat), args.disableColumnFiltering ? null : columnRuleCreator.getDataRequirementsForContext(context));
                StructType resultsSchema = resultSchemas.get(contextName);
                if (resultsSchema == null || resultsSchema.fields().length == 0) {
                    LOG.warn("Context " + contextName + " has no defines configured. Skipping.");
                } else {
                    LOG.info("Evaluating context " + contextName);
                    long contextStartMillis = System.currentTimeMillis();
                    final String outputPath = MapUtils.getRequiredKey(args.outputPaths, context.getName(), "outputPath");
                    JavaPairRDD<Object, List<Row>> rowsByContextId = contextRetriever.retrieveContext(context);
                    CustomMetricSparkPlugin.currentlyEvaluatingContextGauge.setValue(CustomMetricSparkPlugin.currentlyEvaluatingContextGauge.getValue() + 1);
                    JavaPairRDD<Object, Row> resultsByContext = rowsByContextId.flatMapToPair(x -> evaluate(contextName, resultsSchema, x, dataTypeAliases, perContextAccum, errorAccumulator, batchRunTime));
                    writeResults(spark, resultsSchema, resultsByContext, outputPath);
                    long contextEndMillis = System.currentTimeMillis();
                    LOG.info(String.format("Wrote results for context %s to %s", contextName, outputPath));
                    evaluationSummary.addContextCount(contextName, perContextAccum.value());
                    evaluationSummary.addContextRuntime(contextName, contextEndMillis - contextStartMillis);
                    contextAccum.add(1);
                    perContextAccum.reset();
                }
            }
            CustomMetricSparkPlugin.currentlyEvaluatingContextGauge.setValue(0);
            try {
                Boolean metricsEnabledStr = Boolean.valueOf(spark.conf().get("spark.ui.prometheus.enabled"));
                if (metricsEnabledStr) {
                    LOG.info("Prometheus metrics enabled, sleeping for 7 seconds to finish gathering metrics");
                    // sleep for over 5 seconds because Prometheus only polls
                    // every 5 seconds. If spark finishes and goes away immediately after completing,
                    // Prometheus will never be able to poll for the final set of metrics for the spark-submit
                    // The default promtheus config map was changed from 2 minute scrape interval to 5 seconds for spark pods
                    Thread.sleep(7000);
                } else {
                    LOG.info("Prometheus metrics not enabled");
                }
            } catch (NoSuchElementException e) {
                LOG.info("spark.ui.prometheus.enabled is not set");
            }
            evaluationSummary.setJobStatus(JobStatus.SUCCESS);
        } catch (Exception e) {
            // If we experience an error that would make the program halt, capture the error
            // and report it in the batch summary file
            ByteArrayOutputStream errorDetailStream = new ByteArrayOutputStream();
            try (PrintStream printStream = new PrintStream(errorDetailStream)) {
                printStream.write(e.getMessage().getBytes());
                printStream.write('\n');
                if (e.getCause() != null) {
                    printStream.write(e.getCause().getMessage().getBytes());
                    printStream.write('\n');
                }
                e.printStackTrace(printStream);
                evaluationSummary.setErrorList(Collections.singletonList(new EvaluationError(null, null, null, errorDetailStream.toString())));
            }
            throw e;
        } finally {
            long endTimeMillis = System.currentTimeMillis();
            evaluationSummary.setEndTimeMillis(endTimeMillis);
            evaluationSummary.setRuntimeMillis(endTimeMillis - startTimeMillis);
            if (args.metadataOutputPath != null) {
                if (evaluationSummary.getErrorList() == null) {
                    evaluationSummary.setErrorList(errorAccumulator.value());
                }
                if (CollectionUtils.isNotEmpty(evaluationSummary.getErrorList())) {
                    evaluationSummary.setJobStatus(JobStatus.FAIL);
                }
                evaluationSummary.setTotalContexts(contextAccum.value());
                OutputMetadataWriter writer = getOutputMetadataWriter();
                writer.writeMetadata(evaluationSummary);
            }
        }
    }
}
Also used : SparkSession(org.apache.spark.sql.SparkSession) StructType(org.apache.spark.sql.types.StructType) ContextRetriever(com.ibm.cohort.cql.spark.aggregation.ContextRetriever) CqlToElmTranslator(com.ibm.cohort.cql.translation.CqlToElmTranslator) ContextDefinitions(com.ibm.cohort.cql.spark.aggregation.ContextDefinitions) ZonedDateTime(java.time.ZonedDateTime) List(java.util.List) ArrayList(java.util.ArrayList) DefaultDatasetRetriever(com.ibm.cohort.cql.spark.data.DefaultDatasetRetriever) PrintStream(java.io.PrintStream) SerializableConfiguration(org.apache.spark.util.SerializableConfiguration) SparkTypeConverter(com.ibm.cohort.cql.spark.data.SparkTypeConverter) ByteArrayOutputStream(java.io.ByteArrayOutputStream) ContextDefinition(com.ibm.cohort.cql.spark.aggregation.ContextDefinition) FileNotFoundException(java.io.FileNotFoundException) NoSuchElementException(java.util.NoSuchElementException) IOException(java.io.IOException) LongAccumulator(org.apache.spark.util.LongAccumulator) EvaluationSummary(com.ibm.cohort.cql.spark.metadata.EvaluationSummary) HadoopPathOutputMetadataWriter(com.ibm.cohort.cql.spark.metadata.HadoopPathOutputMetadataWriter) OutputMetadataWriter(com.ibm.cohort.cql.spark.metadata.OutputMetadataWriter) ColumnRuleCreator(com.ibm.cohort.cql.spark.aggregation.ColumnRuleCreator) SparkOutputColumnEncoder(com.ibm.cohort.cql.spark.data.SparkOutputColumnEncoder) EvaluationError(com.ibm.cohort.cql.spark.errors.EvaluationError) SparkDataRow(com.ibm.cohort.cql.spark.data.SparkDataRow) DataRow(com.ibm.cohort.datarow.model.DataRow) Row(org.apache.spark.sql.Row) NoSuchElementException(java.util.NoSuchElementException)

Example 4 with ContextDefinition

use of com.ibm.cohort.cql.spark.aggregation.ContextDefinition in project quality-measure-and-cohort-service by Alvearie.

the class SparkSchemaCreator method getDataTypeForContextKey.

private Tuple2<String, DataType> getDataTypeForContextKey(String contextName, Set<Tuple2<String, String>> usingInfos) {
    ContextDefinition contextDefinition = contextDefinitions.getContextDefinitionByName(contextName);
    String primaryDataType = contextDefinition.getPrimaryDataType();
    String primaryKeyColumn = contextDefinition.getPrimaryKeyColumn();
    DataType keyType = null;
    ModelManager modelManager = translator.newModelManager();
    // Try to find the key column's type information from a single model info.
    for (Tuple2<String, String> usingInfo : usingInfos) {
        VersionedIdentifier modelInfoIdentifier = new VersionedIdentifier().withId(usingInfo._1()).withVersion(usingInfo._2());
        ModelInfo modelInfo = modelManager.getModelInfoLoader().getModelInfo(modelInfoIdentifier);
        // Look for a ClassInfo element matching primaryDataType for the context
        List<ClassInfo> classInfos = getClassInfos(primaryDataType, modelInfo);
        if (!classInfos.isEmpty()) {
            if (classInfos.size() == 1) {
                ClassInfo classInfo = classInfos.get(0);
                List<ClassInfoElement> elements = classInfo.getElement().stream().filter(x -> x.getName().equals(primaryKeyColumn)).collect(Collectors.toList());
                // check base type
                String baseType = classInfo.getBaseType();
                if (classInfo.getBaseType() != null) {
                    List<ClassInfo> baseClassInfos = getClassInfos(baseType, modelInfo);
                    baseClassInfos.stream().map(ClassInfo::getElement).flatMap(List::stream).filter(element -> element.getName().equals(primaryKeyColumn)).forEach(elements::add);
                }
                // check choice types
                Collection<String> choiceTypes = ModelUtils.getChoiceTypeNames(classInfo);
                choiceTypes.stream().map(type -> getClassInfos(type, modelInfo)).flatMap(List::stream).map(ClassInfo::getElement).flatMap(List::stream).filter(element -> element.getName().equals(primaryKeyColumn)).findFirst().ifPresent(elements::add);
                // A future ModelInfo file may contain the information
                if (elements.isEmpty()) {
                    continue;
                } else if (elements.size() == 1) {
                    String elementType = elements.get(0).getElementType();
                    // store it
                    if (keyType == null) {
                        keyType = getSparkTypeForSystemValue(elementType);
                    } else {
                        throw new IllegalArgumentException("Multiple definitions found for " + primaryDataType + "." + primaryKeyColumn + " in the provided ModelInfo files. Cannot infer key type for context: " + contextName);
                    }
                } else if (elements.size() > 1) {
                    throw new IllegalArgumentException("ModelInfo " + modelInfoIdentifier + " contains multiple element definitions for " + primaryKeyColumn + " for type " + primaryDataType);
                }
            } else {
                throw new IllegalArgumentException("ModelInfo " + modelInfoIdentifier + " contains multiple definitions for type " + primaryDataType);
            }
        }
    }
    if (keyType == null) {
        throw new IllegalArgumentException("Could not locate type information for " + primaryDataType + "." + primaryKeyColumn + " in the provided ModelInfo files. Cannot infer key type for context: " + contextName);
    }
    return new Tuple2<>(contextDefinition.getPrimaryKeyColumn(), keyType);
}
Also used : DataType(org.apache.spark.sql.types.DataType) ModelInfo(org.hl7.elm_modelinfo.r1.ModelInfo) CqlToElmTranslator(com.ibm.cohort.cql.translation.CqlToElmTranslator) HashMap(java.util.HashMap) Format(com.ibm.cohort.cql.library.Format) ExpressionDef(org.cqframework.cql.elm.execution.ExpressionDef) ClassInfo(org.hl7.elm_modelinfo.r1.ClassInfo) HashSet(java.util.HashSet) ContextDefinition(com.ibm.cohort.cql.spark.aggregation.ContextDefinition) Map(java.util.Map) CqlLibraryReader(org.opencds.cqf.cql.engine.execution.CqlLibraryReader) CqlEvaluationRequest(com.ibm.cohort.cql.evaluation.CqlEvaluationRequest) ModelUtils(com.ibm.cohort.cql.spark.optimizer.ModelUtils) DataTypes(org.apache.spark.sql.types.DataTypes) StructField(org.apache.spark.sql.types.StructField) StructType(org.apache.spark.sql.types.StructType) ModelManager(org.cqframework.cql.cql2elm.ModelManager) Collection(java.util.Collection) VersionedIdentifier(org.hl7.elm.r1.VersionedIdentifier) Set(java.util.Set) CqlLibrary(com.ibm.cohort.cql.library.CqlLibrary) CqlLibraryProvider(com.ibm.cohort.cql.library.CqlLibraryProvider) Tuple2(scala.Tuple2) Collectors(java.util.stream.Collectors) CqlLibraryDescriptor(com.ibm.cohort.cql.library.CqlLibraryDescriptor) List(java.util.List) CqlEvaluationRequests(com.ibm.cohort.cql.evaluation.CqlEvaluationRequests) Library(org.cqframework.cql.elm.execution.Library) ClassInfoElement(org.hl7.elm_modelinfo.r1.ClassInfoElement) QName(javax.xml.namespace.QName) ContextDefinitions(com.ibm.cohort.cql.spark.aggregation.ContextDefinitions) ModelInfo(org.hl7.elm_modelinfo.r1.ModelInfo) ContextDefinition(com.ibm.cohort.cql.spark.aggregation.ContextDefinition) ModelManager(org.cqframework.cql.cql2elm.ModelManager) ClassInfoElement(org.hl7.elm_modelinfo.r1.ClassInfoElement) VersionedIdentifier(org.hl7.elm.r1.VersionedIdentifier) Tuple2(scala.Tuple2) DataType(org.apache.spark.sql.types.DataType) List(java.util.List) ClassInfo(org.hl7.elm_modelinfo.r1.ClassInfo)

Aggregations

ContextDefinition (com.ibm.cohort.cql.spark.aggregation.ContextDefinition)4 ContextDefinitions (com.ibm.cohort.cql.spark.aggregation.ContextDefinitions)3 CqlToElmTranslator (com.ibm.cohort.cql.translation.CqlToElmTranslator)3 CqlEvaluationRequest (com.ibm.cohort.cql.evaluation.CqlEvaluationRequest)2 CqlEvaluationRequests (com.ibm.cohort.cql.evaluation.CqlEvaluationRequests)2 CqlLibraryProvider (com.ibm.cohort.cql.library.CqlLibraryProvider)2 Format (com.ibm.cohort.cql.library.Format)2 ColumnRuleCreator (com.ibm.cohort.cql.spark.aggregation.ColumnRuleCreator)2 ContextRetriever (com.ibm.cohort.cql.spark.aggregation.ContextRetriever)2 DefaultDatasetRetriever (com.ibm.cohort.cql.spark.data.DefaultDatasetRetriever)2 SparkDataRow (com.ibm.cohort.cql.spark.data.SparkDataRow)2 SparkOutputColumnEncoder (com.ibm.cohort.cql.spark.data.SparkOutputColumnEncoder)2 SparkTypeConverter (com.ibm.cohort.cql.spark.data.SparkTypeConverter)2 EvaluationError (com.ibm.cohort.cql.spark.errors.EvaluationError)2 EvaluationSummary (com.ibm.cohort.cql.spark.metadata.EvaluationSummary)2 HadoopPathOutputMetadataWriter (com.ibm.cohort.cql.spark.metadata.HadoopPathOutputMetadataWriter)2 OutputMetadataWriter (com.ibm.cohort.cql.spark.metadata.OutputMetadataWriter)2 ModelUtils (com.ibm.cohort.cql.spark.optimizer.ModelUtils)2 List (java.util.List)2 StructType (org.apache.spark.sql.types.StructType)2