Examples with UploadFieldType org.sagebionetworks.bridge.models.upload.UploadFieldType used on opensource projects

Example 1 with UploadFieldType

use of org.sagebionetworks.bridge.models.upload.UploadFieldType in project BridgeServer2 by Sage-Bionetworks.

the class UploadSchemaService method createUploadSchemaFromSurvey.

/**
 * <p>
 * Creates an upload schema from a survey. This is generally called when a survey is published, to
 * create the corresponding upload schema, so that health data records can be created from survey responses.
 * This method will also persist the schema to the backing store.
 * <p>
 * If newSchemaRev is true, this method will always create a new schema revision. If false, it will attempt to
 * modify the existing schema revision. However, if the schema revisions are not compatible, it will fall back to
 * creating a new schema revision.
 * </p>
 */
public UploadSchema createUploadSchemaFromSurvey(String appId, Survey survey, boolean newSchemaRev) {
    // https://sagebionetworks.jira.com/browse/BRIDGE-1698 - If the existing Schema ID points to a different survey
    // or a non-survey, this is an error. Having multiple surveys point to the same schema ID causes really bad
    // things to happen, and we need to prevent it.
    String schemaId = survey.getIdentifier();
    UploadSchema oldSchema = getUploadSchemaNoThrow(appId, schemaId);
    if (oldSchema != null) {
        if (oldSchema.getSchemaType() != UploadSchemaType.IOS_SURVEY || !Objects.equals(oldSchema.getSurveyGuid(), survey.getGuid())) {
            throw new BadRequestException("Survey with identifier " + schemaId + " conflicts with schema with the same ID. Please use a different survey identifier.");
        }
    }
    // the same survey.
    if (!newSchemaRev && oldSchema != null) {
        // Check that the old schema already has the answers field.
        List<UploadFieldDefinition> oldFieldDefList = oldSchema.getFieldDefinitions();
        UploadFieldDefinition answersFieldDef = getElement(oldFieldDefList, UploadFieldDefinition::getName, FIELD_ANSWERS).orElse(null);
        if (answersFieldDef == null) {
            // Old schema doesn't have the
            List<UploadFieldDefinition> newFieldDefList = new ArrayList<>(oldFieldDefList);
            newFieldDefList.add(UploadUtil.ANSWERS_FIELD_DEF);
            addSurveySchemaMetadata(oldSchema, survey);
            oldSchema.setFieldDefinitions(newFieldDefList);
            return updateSchemaRevisionV4(appId, schemaId, oldSchema.getRevision(), oldSchema);
        }
        // Answers field needs to be either
        // (a) an attachment (Large Text or normal)
        // (b) a string with isUnboundedLength=true
        UploadFieldType fieldType = answersFieldDef.getType();
        if (fieldType == UploadFieldType.LARGE_TEXT_ATTACHMENT || UploadFieldType.ATTACHMENT_TYPE_SET.contains(fieldType) || (UploadFieldType.STRING_TYPE_SET.contains(fieldType) && Boolean.TRUE.equals(answersFieldDef.isUnboundedText()))) {
            // The old schema works for the new survey. However, we want to ensure the old schema points to the
            // latest version of the survey. Update survey metadata in the schema.
            addSurveySchemaMetadata(oldSchema, survey);
            return updateSchemaRevisionV4(appId, schemaId, oldSchema.getRevision(), oldSchema);
        }
    // If execution gets this far, that means we have a schema with an "answers" field that's not compatible.
    // At this point, we go into the branch that creates a new schema, below.
    }
    // We were unable to reconcile this with the existing schema. Create a new schema. (Create API will
    // automatically bump the rev number if an old schema revision exists.)
    UploadSchema schemaToCreate = UploadSchema.create();
    addSurveySchemaMetadata(schemaToCreate, survey);
    schemaToCreate.setFieldDefinitions(ImmutableList.of(UploadUtil.ANSWERS_FIELD_DEF));
    return createSchemaRevisionV4(appId, schemaToCreate);
}

Example 2 with UploadFieldType

use of org.sagebionetworks.bridge.models.upload.UploadFieldType in project BridgeServer2 by Sage-Bionetworks.

the class UploadUtilTest method calculateFieldSizeSimpleField.

@Test
public void calculateFieldSizeSimpleField() {
    // { fieldType, expectedBytes }
    Object[][] testCaseArray = { { UploadFieldType.ATTACHMENT_BLOB, 20 }, { UploadFieldType.ATTACHMENT_CSV, 20 }, { UploadFieldType.ATTACHMENT_JSON_BLOB, 20 }, { UploadFieldType.ATTACHMENT_JSON_TABLE, 20 }, { UploadFieldType.ATTACHMENT_V2, 20 }, { UploadFieldType.BOOLEAN, 5 }, { UploadFieldType.CALENDAR_DATE, 30 }, { UploadFieldType.DURATION_V2, 72 }, { UploadFieldType.FLOAT, 23 }, { UploadFieldType.INT, 20 }, { UploadFieldType.LARGE_TEXT_ATTACHMENT, 3000 }, { UploadFieldType.TIME_V2, 36 } };
    for (Object[] testCase : testCaseArray) {
        UploadFieldType fieldType = (UploadFieldType) testCase[0];
        int expectedBytes = (int) testCase[1];
        UploadFieldDefinition fieldDef = new UploadFieldDefinition.Builder().withName("field").withType(fieldType).build();
        UploadFieldSize fieldSize = UploadUtil.calculateFieldSize(ImmutableList.of(fieldDef));
        assertEquals(fieldSize.getNumBytes(), expectedBytes, "bytes for " + fieldType);
        assertEquals(fieldSize.getNumColumns(), 1, "columns for " + fieldType);
    }
}

Example 3 with UploadFieldType

use of org.sagebionetworks.bridge.models.upload.UploadFieldType in project BridgeServer2 by Sage-Bionetworks.

the class UploadFieldDefinitionListValidator method validate.

/**
 * Validates a List of UploadFieldDefinitions.
 *
 * @param fieldDefList
 *         field definition list to validate
 * @param errors
 *         validation errors are written to this
 * @param attributeName
 *         the parent class's attribute name for the field definition list, as would be reported in errors
 */
public void validate(List<UploadFieldDefinition> fieldDefList, Errors errors, String attributeName) {
    if (fieldDefList == null || fieldDefList.isEmpty()) {
        // App.uploadMetadataFieldDefinitions.
        return;
    }
    // Keep track of field names seen. This list may include duplicates, which we validate in a later step.
    List<String> fieldNameList = new ArrayList<>();
    for (int i = 0; i < fieldDefList.size(); i++) {
        UploadFieldDefinition fieldDef = fieldDefList.get(i);
        String fieldDefinitionKey = attributeName + "[" + i + "]";
        if (fieldDef == null) {
            errors.rejectValue(fieldDefinitionKey, "is required");
        } else {
            errors.pushNestedPath(fieldDefinitionKey);
            String fieldName = fieldDef.getName();
            if (StringUtils.isBlank(fieldName)) {
                errors.rejectValue("name", "is required");
            } else {
                fieldNameList.add(fieldName);
                // Validate field name.
                if (!UploadUtil.isValidSchemaFieldName(fieldName)) {
                    errors.rejectValue("name", String.format(UploadUtil.INVALID_FIELD_NAME_ERROR_MESSAGE, fieldName));
                }
            }
            UploadFieldType fieldType = fieldDef.getType();
            if (fieldType == null) {
                errors.rejectValue("type", "is required");
            }
            if (fieldType == UploadFieldType.MULTI_CHOICE) {
                List<String> multiChoiceAnswerList = fieldDef.getMultiChoiceAnswerList();
                if (multiChoiceAnswerList == null || multiChoiceAnswerList.isEmpty()) {
                    errors.rejectValue("multiChoiceAnswerList", "must be specified for MULTI_CHOICE field " + fieldName);
                } else {
                    // Multi-Choice fields create extra "sub-field" columns, and we need to check for
                    // potential name collisions.
                    int numAnswers = multiChoiceAnswerList.size();
                    for (int j = 0; j < numAnswers; j++) {
                        String oneAnswer = multiChoiceAnswerList.get(j);
                        fieldNameList.add(fieldName + MULTI_CHOICE_FIELD_SEPARATOR + oneAnswer);
                        // Validate choice answer name.
                        if (!UploadUtil.isValidAnswerChoice(oneAnswer)) {
                            errors.rejectValue("multiChoice[" + j + "]", String.format(UploadUtil.INVALID_ANSWER_CHOICE_ERROR_MESSAGE, oneAnswer));
                        }
                    }
                }
                if (Boolean.TRUE.equals(fieldDef.getAllowOtherChoices())) {
                    // Similarly, there's an "other" field.
                    fieldNameList.add(fieldName + OTHER_CHOICE_FIELD_SUFFIX);
                }
            } else if (fieldType == UploadFieldType.TIMESTAMP) {
                // Timestamp fields also generate a second subfield for timezone. Need to check for name
                // collisions here too.
                fieldNameList.add(fieldName + TIME_ZONE_FIELD_SUFFIX);
            }
            if (fieldDef.isUnboundedText() != null && fieldDef.isUnboundedText() && fieldDef.getMaxLength() != null) {
                errors.rejectValue("unboundedText", "cannot specify unboundedText=true with a maxLength");
            }
            errors.popNestedPath();
        }
    }
    // Check for duplicate field names. Dupe set is a tree set so our error messages are in a predictable
    // alphabetical order.
    Set<String> seenFieldNameSet = new HashSet<>();
    Set<String> dupeFieldNameSet = new TreeSet<>();
    for (String oneFieldName : fieldNameList) {
        if (seenFieldNameSet.contains(oneFieldName)) {
            dupeFieldNameSet.add(oneFieldName);
        } else {
            seenFieldNameSet.add(oneFieldName);
        }
    }
    if (!dupeFieldNameSet.isEmpty()) {
        errors.rejectValue(attributeName, "conflict in field names or sub-field names: " + BridgeUtils.COMMA_SPACE_JOINER.join(dupeFieldNameSet));
    }
}

Example 4 with UploadFieldType

use of org.sagebionetworks.bridge.models.upload.UploadFieldType in project BridgeServer2 by Sage-Bionetworks.

the class StrictValidationHandler method validateAllFields.

/**
 * Given the schema, the attachments (all we need are names), and the JSON data nodes, we validate the data against
 * the schema.
 */
private static List<String> validateAllFields(List<UploadFieldDefinition> fieldDefList, JsonNode recordDataNode) {
    // walk the field definitions and validate fields
    List<String> errorList = new ArrayList<>();
    for (UploadFieldDefinition oneFieldDef : fieldDefList) {
        String fieldName = oneFieldDef.getName();
        UploadFieldType fieldType = oneFieldDef.getType();
        boolean isRequired = oneFieldDef.isRequired();
        if (UploadFieldType.ATTACHMENT_TYPE_SET.contains(fieldType)) {
            // required and present. Specifically, if it's required and it's not present, then that's an error.
            if (isRequired && !recordDataNode.hasNonNull(fieldName)) {
                errorList.add("Required attachment field " + fieldName + " missing");
            }
        } else {
            JsonNode fieldValueNode = recordDataNode.get(fieldName);
            if (fieldValueNode != null && !fieldValueNode.isNull()) {
                // Canonicalize the field.
                CanonicalizationResult canonicalizationResult = UploadUtil.canonicalize(fieldValueNode, fieldType);
                if (canonicalizationResult.isValid()) {
                    JsonNode canonicalizedValueNode = canonicalizationResult.getCanonicalizedValueNode();
                    // Special case: MULTI_CHOICE value validation (unless it allows other choices)
                    if (fieldType == UploadFieldType.MULTI_CHOICE && !Boolean.TRUE.equals(oneFieldDef.getAllowOtherChoices())) {
                        // noinspection ConstantConditions
                        Set<String> validAnswerSet = new HashSet<>(oneFieldDef.getMultiChoiceAnswerList());
                        int numAnswers = canonicalizedValueNode.size();
                        for (int i = 0; i < numAnswers; i++) {
                            String answer = canonicalizedValueNode.get(i).textValue();
                            if (!validAnswerSet.contains(answer)) {
                                errorList.add("Multi-Choice field " + fieldName + " contains invalid answer " + answer);
                            }
                        }
                    }
                    // Write the canonicalization back into the field data map.
                    ((ObjectNode) recordDataNode).set(fieldName, canonicalizedValueNode);
                } else {
                    errorList.add("Canonicalization failed for field " + fieldName + ": " + canonicalizationResult.getErrorMessage());
                }
            } else if (isRequired) {
                errorList.add("Required field " + fieldName + " missing");
            }
        }
    }
    return errorList;
}

Example 5 with UploadFieldType

use of org.sagebionetworks.bridge.models.upload.UploadFieldType in project BridgeServer2 by Sage-Bionetworks.

the class UploadUtil method calculateFieldSize.

/**
 * Calculates the total field size for the list of field definitions.
 */
public static UploadFieldSize calculateFieldSize(List<UploadFieldDefinition> fieldDefList) {
    int numBytes = 0;
    int numColumns = 0;
    for (UploadFieldDefinition fieldDef : fieldDefList) {
        UploadFieldType fieldType = fieldDef.getType();
        if (fieldType == null) {
            // field. The validator will throw anyway.
            continue;
        }
        // Calculate number of bytes.
        if (VARIABLE_LENGTH_STRING_TYPE_SET.contains(fieldType)) {
            // Special case for string types, since this scales based on max length.
            if (Boolean.TRUE.equals(fieldDef.isUnboundedText())) {
                numBytes += SYNAPSE_LARGE_TEXT_BYTE_SIZE;
            } else {
                // Synapse string fields cost 3 bytes per char.
                int maxCharLength = fieldDef.getMaxLength() != null ? fieldDef.getMaxLength() : DEFAULT_MAX_LENGTH;
                numBytes += 3 * maxCharLength;
            }
        } else if (fieldType == UploadFieldType.MULTI_CHOICE) {
            // Multi-choice has a boolean column (5 bytes) for each answer. (Note that the field def builder
            // guarantees that the multi-choice answer list is not null, though it might be empty.
            numBytes += 5 * fieldDef.getMultiChoiceAnswerList().size();
            // Multi-choice also adds a LargeText (3000 bytes) if allowOtherChoices is true.
            if (Boolean.TRUE.equals(fieldDef.getAllowOtherChoices())) {
                numBytes += SYNAPSE_LARGE_TEXT_BYTE_SIZE;
            }
        } else if (SYNAPSE_BYTE_SIZE_BY_TYPE.containsKey(fieldType)) {
            numBytes += SYNAPSE_BYTE_SIZE_BY_TYPE.get(fieldType);
        } else {
            throw new BridgeServiceException("Couldn't get byte size for field type " + fieldType);
        }
        // Calculate number of columns.
        if (fieldType == UploadFieldType.MULTI_CHOICE) {
            // Multi-choice fields have a boolean column for each answer.
            if (fieldDef.getMultiChoiceAnswerList() != null) {
                numColumns += fieldDef.getMultiChoiceAnswerList().size();
            }
            // Multi-choice also adds a LargeText if allowOtherChoices is true.
            if (Boolean.TRUE.equals(fieldDef.getAllowOtherChoices())) {
                numColumns++;
            }
        } else if (fieldType == UploadFieldType.TIMESTAMP) {
            // Timestamp has two columns, one for epoch time, one for timezone.
            numColumns += 2;
        } else {
            // Every other field type creates only one column.
            numColumns++;
        }
    }
    return new UploadFieldSize(numBytes, numColumns);
}