Examples with FieldDescriptor org.apache.drill.common.scanner.persistence.FieldDescriptor used on opensource projects

Example 1 with FieldDescriptor

use of org.apache.drill.common.scanner.persistence.FieldDescriptor in project drill by apache.

the class TestClassPathScanner method test.

@Test
public void test() throws Exception {
    ScanResult result = ClassPathScanner.fromPrescan(DrillConfig.create());
    // if the build has run properly. BuildTimeScan.REGISTRY_FILE was created with a prescan
    //    assertListEqualsUnordered(result.getPrescannedPackages(),
    //      "org.apache.drill.common.logical",
    //      "org.apache.drill.exec.expr",
    //      "org.apache.drill.exec.physical.base",
    //      "org.apache.drill.exec.expr.fn.impl",
    //      "org.apache.drill.exec.physical.impl",
    //      "org.apache.drill.exec.rpc.user.security",
    //      "org.apache.drill.exec.store",
    //      "org.apache.drill.exec.store.mock",
    //      "org.apache.drill.exec.physical.config",
    //      "org.apache.drill.storage"
    //    );
    //    // this is added in the unit test folder that was not scanned so far
    //    assertListEqualsUnordered(result.getScannedPackages(),
    //      "org.apache.drill.exec.testing",
    //      "org.apache.drill.exec.fn.impl.testing",
    //      "org.apache.drill.exec.rpc.user.security.testing"
    //    );
    List<AnnotatedClassDescriptor> functions = result.getAnnotatedClasses();
    Set<String> scanned = new HashSet<>();
    AnnotatedClassDescriptor functionRandomBigIntGauss = null;
    for (AnnotatedClassDescriptor function : functions) {
        assertTrue(function.getClassName() + " scanned twice", scanned.add(function.getClassName()));
        if (function.getClassName().equals(RandomBigIntGauss.class.getName())) {
            functionRandomBigIntGauss = function;
        }
    }
    if (functionRandomBigIntGauss == null) {
        Assert.fail("functionRandomBigIntGauss not found");
    }
    // TODO: use Andrew's randomized test framework to verify a subset of the functions
    for (AnnotatedClassDescriptor function : functions) {
        Class<?> c = Class.forName(function.getClassName(), false, this.getClass().getClassLoader());
        Field[] fields = c.getDeclaredFields();
        assertEquals("fields count for " + function, fields.length, function.getFields().size());
        for (int i = 0; i < fields.length; i++) {
            FieldDescriptor fieldDescriptor = function.getFields().get(i);
            Field field = fields[i];
            assertEquals("Class fields:\n" + Arrays.toString(fields) + "\n != \nDescriptor fields:\n" + function.getFields(), field.getName(), fieldDescriptor.getName());
            verifyAnnotations(field.getDeclaredAnnotations(), fieldDescriptor.getAnnotations());
            assertEquals(field.getType(), fieldDescriptor.getFieldClass());
        }
        Annotation[] annotations = c.getDeclaredAnnotations();
        List<AnnotationDescriptor> scannedAnnotations = function.getAnnotations();
        verifyAnnotations(annotations, scannedAnnotations);
        FunctionTemplate bytecodeAnnotation = function.getAnnotationProxy(FunctionTemplate.class);
        FunctionTemplate reflectionAnnotation = c.getAnnotation(FunctionTemplate.class);
        assertEquals(reflectionAnnotation.name(), bytecodeAnnotation.name());
        assertArrayEquals(reflectionAnnotation.names(), bytecodeAnnotation.names());
        assertEquals(reflectionAnnotation.scope(), bytecodeAnnotation.scope());
        assertEquals(reflectionAnnotation.nulls(), bytecodeAnnotation.nulls());
        assertEquals(reflectionAnnotation.isBinaryCommutative(), bytecodeAnnotation.isBinaryCommutative());
        assertEquals(reflectionAnnotation.desc(), bytecodeAnnotation.desc());
        assertEquals(reflectionAnnotation.costCategory(), bytecodeAnnotation.costCategory());
    }
    for (String baseType : result.getScannedClasses()) {
        validateType(result, baseType);
    }
    assertTrue(result.getImplementations(PhysicalOperator.class).size() > 0);
    assertTrue(result.getImplementations(DrillFunc.class).size() > 0);
}

Example 2 with FieldDescriptor

use of org.apache.drill.common.scanner.persistence.FieldDescriptor in project drill by axbaretto.

the class TestClassPathScanner method test.

@Test
public void test() throws Exception {
    ScanResult result = ClassPathScanner.fromPrescan(DrillConfig.create());
    List<AnnotatedClassDescriptor> functions = result.getAnnotatedClasses();
    Set<String> scanned = new HashSet<>();
    AnnotatedClassDescriptor functionRandomBigIntGauss = null;
    for (AnnotatedClassDescriptor function : functions) {
        assertTrue(function.getClassName() + " scanned twice", scanned.add(function.getClassName()));
        if (function.getClassName().equals(RandomBigIntGauss.class.getName())) {
            functionRandomBigIntGauss = function;
        }
    }
    if (functionRandomBigIntGauss == null) {
        Assert.fail("functionRandomBigIntGauss not found");
    }
    // TODO: use Andrew's randomized test framework to verify a subset of the functions
    for (AnnotatedClassDescriptor function : functions) {
        Class<?> c = Class.forName(function.getClassName(), false, this.getClass().getClassLoader());
        Field[] fields = c.getDeclaredFields();
        assertEquals("fields count for " + function, fields.length, function.getFields().size());
        for (int i = 0; i < fields.length; i++) {
            FieldDescriptor fieldDescriptor = function.getFields().get(i);
            Field field = fields[i];
            assertEquals("Class fields:\n" + Arrays.toString(fields) + "\n != \nDescriptor fields:\n" + function.getFields(), field.getName(), fieldDescriptor.getName());
            verifyAnnotations(field.getDeclaredAnnotations(), fieldDescriptor.getAnnotations());
            assertEquals(field.getType(), fieldDescriptor.getFieldClass());
        }
        Annotation[] annotations = c.getDeclaredAnnotations();
        List<AnnotationDescriptor> scannedAnnotations = function.getAnnotations();
        verifyAnnotations(annotations, scannedAnnotations);
        FunctionTemplate bytecodeAnnotation = function.getAnnotationProxy(FunctionTemplate.class);
        FunctionTemplate reflectionAnnotation = c.getAnnotation(FunctionTemplate.class);
        assertEquals(reflectionAnnotation.name(), bytecodeAnnotation.name());
        assertArrayEquals(reflectionAnnotation.names(), bytecodeAnnotation.names());
        assertEquals(reflectionAnnotation.scope(), bytecodeAnnotation.scope());
        assertEquals(reflectionAnnotation.nulls(), bytecodeAnnotation.nulls());
        assertEquals(reflectionAnnotation.isBinaryCommutative(), bytecodeAnnotation.isBinaryCommutative());
        assertEquals(reflectionAnnotation.desc(), bytecodeAnnotation.desc());
        assertEquals(reflectionAnnotation.costCategory(), bytecodeAnnotation.costCategory());
    }
    for (String baseType : result.getScannedClasses()) {
        validateType(result, baseType);
    }
    assertTrue(result.getImplementations(PhysicalOperator.class).size() > 0);
    assertTrue(result.getImplementations(DrillFunc.class).size() > 0);
}

Example 3 with FieldDescriptor

use of org.apache.drill.common.scanner.persistence.FieldDescriptor in project drill by axbaretto.

the class FunctionConverter method getHolder.

public <T extends DrillFunc> DrillFuncHolder getHolder(AnnotatedClassDescriptor func, ClassLoader classLoader) {
    FunctionTemplate template = func.getAnnotationProxy(FunctionTemplate.class);
    if (template == null) {
        return failure("Class does not declare FunctionTemplate annotation.", func);
    }
    String name = template.name();
    List<String> names = Arrays.asList(template.names());
    if (name.isEmpty() && names.isEmpty()) {
        // none set
        return failure("Must define 'name' or 'names'", func);
    }
    if (!name.isEmpty() && !names.isEmpty()) {
        // both are set
        return failure("Must use only one annotations 'name' or 'names', not both", func);
    }
    // start by getting field information.
    List<ValueReference> params = Lists.newArrayList();
    List<WorkspaceReference> workspaceFields = Lists.newArrayList();
    ValueReference outputField = null;
    for (FieldDescriptor field : func.getFields()) {
        Param param = field.getAnnotationProxy(Param.class);
        Output output = field.getAnnotationProxy(Output.class);
        Workspace workspace = field.getAnnotationProxy(Workspace.class);
        Inject inject = field.getAnnotationProxy(Inject.class);
        Annotation[] annotations = { param, output, workspace, inject };
        int annotationCount = 0;
        for (Annotation annotationDescriptor : annotations) {
            if (annotationDescriptor != null) {
                annotationCount += 1;
            }
        }
        if (annotationCount == 0) {
            return failure("The field must be either a @Param, @Output, @Inject or @Workspace field.", func, field);
        } else if (annotationCount > 1) {
            return failure("The field must be only one of @Param, @Output, @Inject or @Workspace. It currently has more than one of these annotations.", func, field);
        }
        // TODO(Julien): verify there are a few of those and we can load them
        Class<?> fieldClass = field.getFieldClass();
        if (param != null || output != null) {
            // Special processing for @Param FieldReader
            if (param != null && FieldReader.class.isAssignableFrom(fieldClass)) {
                params.add(ValueReference.createFieldReaderRef(field.getName()));
                continue;
            }
            // Special processing for @Output ComplexWriter
            if (output != null && ComplexWriter.class.isAssignableFrom(fieldClass)) {
                if (outputField != null) {
                    return failure("You've declared more than one @Output field.  You must declare one and only @Output field per Function class.", func, field);
                } else {
                    outputField = ValueReference.createComplexWriterRef(field.getName());
                }
                continue;
            }
            // check that param and output are value holders.
            if (!ValueHolder.class.isAssignableFrom(fieldClass)) {
                return failure(String.format("The field doesn't holds value of type %s which does not implement the ValueHolder interface.  All fields of type @Param or @Output must extend this interface..", fieldClass), func, field);
            }
            // get the type field from the value holder.
            MajorType type = null;
            try {
                type = getStaticFieldValue("TYPE", fieldClass, MajorType.class);
            } catch (Exception e) {
                return failure("Failure while trying to access the ValueHolder's TYPE static variable.  All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, func, field);
            }
            ValueReference p = new ValueReference(type, field.getName());
            if (param != null) {
                p.setConstant(param.constant());
                params.add(p);
            } else {
                if (outputField != null) {
                    return failure("You've declared more than one @Output field.  You must declare one and only @Output field per Function class.", func, field);
                } else {
                    outputField = p;
                }
            }
        } else {
            // workspace work.
            boolean isInject = inject != null;
            if (isInject && UdfUtilities.INJECTABLE_GETTER_METHODS.get(fieldClass) == null) {
                return failure(String.format("A %s cannot be injected into a %s," + " available injectable classes are: %s.", fieldClass, DrillFunc.class.getSimpleName(), Joiner.on(",").join(UdfUtilities.INJECTABLE_GETTER_METHODS.keySet())), func, field);
            }
            WorkspaceReference wsReference = new WorkspaceReference(fieldClass, field.getName(), isInject);
            if (!isInject && template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(fieldClass)) {
                return failure(String.format("Aggregate function '%s' workspace variable '%s' is of type '%s'. Please change it to Holder type.", func.getClassName(), field.getName(), fieldClass), func, field);
            }
            // If the workspace var is of Holder type, get its MajorType and assign to WorkspaceReference.
            if (ValueHolder.class.isAssignableFrom(fieldClass)) {
                MajorType majorType = null;
                try {
                    majorType = getStaticFieldValue("TYPE", fieldClass, MajorType.class);
                } catch (Exception e) {
                    return failure("Failure while trying to access the ValueHolder's TYPE static variable.  All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, func, field);
                }
                wsReference.setMajorType(majorType);
            }
            workspaceFields.add(wsReference);
        }
    }
    if (outputField == null) {
        return failure("This function declares zero output fields.  A function must declare one output field.", func);
    }
    FunctionInitializer initializer = new FunctionInitializer(func.getClassName(), classLoader);
    try {
        // return holder
        ValueReference[] ps = params.toArray(new ValueReference[params.size()]);
        WorkspaceReference[] works = workspaceFields.toArray(new WorkspaceReference[workspaceFields.size()]);
        FunctionAttributes functionAttributes = new FunctionAttributes(template, ps, outputField, works);
        switch(template.scope()) {
            case POINT_AGGREGATE:
                return new DrillAggFuncHolder(functionAttributes, initializer);
            case SIMPLE:
                return outputField.isComplexWriter() ? new DrillComplexWriterFuncHolder(functionAttributes, initializer) : new DrillSimpleFuncHolder(functionAttributes, initializer);
            case HOLISTIC_AGGREGATE:
            case RANGE_AGGREGATE:
            default:
                return failure("Unsupported Function Type.", func);
        }
    } catch (Exception | NoSuchFieldError | AbstractMethodError ex) {
        return failure("Failure while creating function holder.", ex, func);
    }
}

Example 4 with FieldDescriptor

use of org.apache.drill.common.scanner.persistence.FieldDescriptor in project drill by apache.

the class TestClassPathScanner method testFunctionTemplates.

@Test
public void testFunctionTemplates() throws Exception {
    List<AnnotatedClassDescriptor> functions = result.getAnnotatedClasses(FunctionTemplate.class.getName());
    Set<String> scanned = new HashSet<>();
    AnnotatedClassDescriptor functionRandomBigIntGauss = null;
    for (AnnotatedClassDescriptor function : functions) {
        assertTrue(function.getClassName() + " scanned twice", scanned.add(function.getClassName()));
        if (function.getClassName().equals(RandomBigIntGauss.class.getName())) {
            functionRandomBigIntGauss = function;
        }
    }
    if (functionRandomBigIntGauss == null) {
        Assert.fail("functionRandomBigIntGauss not found");
    }
    // TODO: use Andrew's randomized test framework to verify a subset of the functions
    for (AnnotatedClassDescriptor function : functions) {
        Class<?> c = Class.forName(function.getClassName(), false, this.getClass().getClassLoader());
        Field[] fields = c.getDeclaredFields();
        assertEquals("fields count for " + function, fields.length, function.getFields().size());
        for (int i = 0; i < fields.length; i++) {
            FieldDescriptor fieldDescriptor = function.getFields().get(i);
            Field field = fields[i];
            assertEquals("Class fields:\n" + Arrays.toString(fields) + "\n != \nDescriptor fields:\n" + function.getFields(), field.getName(), fieldDescriptor.getName());
            verifyAnnotations(field.getDeclaredAnnotations(), fieldDescriptor.getAnnotations());
            assertEquals(field.getType(), fieldDescriptor.getFieldClass());
        }
        Annotation[] annotations = c.getDeclaredAnnotations();
        List<AnnotationDescriptor> scannedAnnotations = function.getAnnotations();
        verifyAnnotations(annotations, scannedAnnotations);
        FunctionTemplate bytecodeAnnotation = function.getAnnotationProxy(FunctionTemplate.class);
        assertNotNull(bytecodeAnnotation);
        FunctionTemplate reflectionAnnotation = c.getAnnotation(FunctionTemplate.class);
        assertEquals(reflectionAnnotation.name(), bytecodeAnnotation.name());
        assertArrayEquals(reflectionAnnotation.names(), bytecodeAnnotation.names());
        assertEquals(reflectionAnnotation.scope(), bytecodeAnnotation.scope());
        assertEquals(reflectionAnnotation.nulls(), bytecodeAnnotation.nulls());
        assertEquals(reflectionAnnotation.isBinaryCommutative(), bytecodeAnnotation.isBinaryCommutative());
        assertEquals(reflectionAnnotation.desc(), bytecodeAnnotation.desc());
        assertEquals(reflectionAnnotation.costCategory(), bytecodeAnnotation.costCategory());
    }
    for (String baseType : result.getScannedClasses()) {
        validateType(result, baseType);
    }
    assertTrue(result.getImplementations(PhysicalOperator.class).size() > 0);
    assertTrue(result.getImplementations(DrillFunc.class).size() > 0);
}

Example 5 with FieldDescriptor

use of org.apache.drill.common.scanner.persistence.FieldDescriptor in project drill by apache.

the class FunctionConverter method getHolder.

public DrillFuncHolder getHolder(AnnotatedClassDescriptor func, ClassLoader classLoader) {
    FunctionTemplate template = func.getAnnotationProxy(FunctionTemplate.class);
    if (template == null) {
        return failure("Class does not declare FunctionTemplate annotation.", func);
    }
    String name = template.name();
    List<String> names = Arrays.asList(template.names());
    if (name.isEmpty() && names.isEmpty()) {
        // none set
        return failure("Must define 'name' or 'names'", func);
    }
    if (!name.isEmpty() && !names.isEmpty()) {
        // both are set
        return failure("Must use only one annotations 'name' or 'names', not both", func);
    }
    // start by getting field information.
    List<ValueReference> params = Lists.newArrayList();
    List<WorkspaceReference> workspaceFields = Lists.newArrayList();
    ValueReference outputField = null;
    int varArgsCount = 0;
    for (FieldDescriptor field : func.getFields()) {
        Param param = field.getAnnotationProxy(Param.class);
        Output output = field.getAnnotationProxy(Output.class);
        Workspace workspace = field.getAnnotationProxy(Workspace.class);
        Inject inject = field.getAnnotationProxy(Inject.class);
        Annotation[] annotations = { param, output, workspace, inject };
        int annotationCount = 0;
        for (Annotation annotationDescriptor : annotations) {
            if (annotationDescriptor != null) {
                annotationCount += 1;
            }
        }
        if (annotationCount == 0) {
            return failure("The field must be either a @Param, @Output, @Inject or @Workspace field.", func, field);
        } else if (annotationCount > 1) {
            return failure("The field must be only one of @Param, @Output, @Inject or @Workspace. It currently has more than one of these annotations.", func, field);
        }
        // TODO(Julien): verify there are a few of those and we can load them
        Class<?> fieldClass = field.getFieldClass();
        if (param != null || output != null) {
            if (Object[].class.isAssignableFrom(fieldClass)) {
                fieldClass = fieldClass.getComponentType();
                varArgsCount++;
            } else if (varArgsCount > 0 && param != null) {
                return failure("Vararg should be the last argument in the function.", func, field);
            }
            if (varArgsCount > 1) {
                return failure("Function should contain single vararg argument", func, field);
            }
            // Special processing for @Param FieldReader
            if (param != null && FieldReader.class.isAssignableFrom(fieldClass)) {
                ValueReference fieldReaderRef = ValueReference.createFieldReaderRef(field.getName());
                fieldReaderRef.setVarArg(varArgsCount > 0);
                params.add(fieldReaderRef);
                continue;
            }
            // Special processing for @Output ComplexWriter
            if (output != null && ComplexWriter.class.isAssignableFrom(fieldClass)) {
                if (outputField != null) {
                    return failure("You've declared more than one @Output field.\n" + "You must declare one and only @Output field per Function class.", func, field);
                } else {
                    outputField = ValueReference.createComplexWriterRef(field.getName());
                }
                continue;
            }
            // check that param and output are value holders.
            if (!ValueHolder.class.isAssignableFrom(fieldClass)) {
                return failure(String.format("The field doesn't holds value of type %s which does not implement the ValueHolder or ComplexWriter interfaces.\n" + "All fields of type @Param or @Output must extend this interface.", fieldClass), func, field);
            }
            // get the type field from the value holder.
            MajorType type;
            try {
                type = getStaticFieldValue("TYPE", fieldClass, MajorType.class);
            } catch (Exception e) {
                return failure("Failure while trying to access the ValueHolder's TYPE static variable.  All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, func, field);
            }
            ValueReference p = new ValueReference(type, field.getName());
            if (param != null) {
                p.setConstant(param.constant());
                p.setVarArg(varArgsCount > 0);
                params.add(p);
            } else {
                if (outputField != null) {
                    return failure("You've declared more than one @Output field.  You must declare one and only @Output field per Function class.", func, field);
                } else {
                    outputField = p;
                }
            }
        } else {
            // workspace work.
            boolean isInject = inject != null;
            if (isInject && UdfUtilities.INJECTABLE_GETTER_METHODS.get(fieldClass) == null) {
                return failure(String.format("A %s cannot be injected into a %s," + " available injectable classes are: %s.", fieldClass, DrillFunc.class.getSimpleName(), Joiner.on(",").join(UdfUtilities.INJECTABLE_GETTER_METHODS.keySet())), func, field);
            }
            WorkspaceReference wsReference = new WorkspaceReference(fieldClass, field.getName(), isInject);
            if (!isInject && template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(fieldClass)) {
                return failure(String.format("Aggregate function '%s' workspace variable '%s' is of type '%s'. Please change it to Holder type.", func.getClassName(), field.getName(), fieldClass), func, field);
            }
            // If the workspace var is of Holder type, get its MajorType and assign to WorkspaceReference.
            if (ValueHolder.class.isAssignableFrom(fieldClass)) {
                MajorType majorType;
                try {
                    majorType = getStaticFieldValue("TYPE", fieldClass, MajorType.class);
                } catch (Exception e) {
                    return failure("Failure while trying to access the ValueHolder's TYPE static variable.  All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, func, field);
                }
                wsReference.setMajorType(majorType);
            }
            workspaceFields.add(wsReference);
        }
    }
    if (outputField == null) {
        return failure("This function declares zero output fields.  A function must declare one output field.", func);
    }
    FunctionInitializer initializer = new FunctionInitializer(func.getClassName(), classLoader);
    try {
        // return holder
        ValueReference[] ps = params.toArray(new ValueReference[0]);
        WorkspaceReference[] works = workspaceFields.toArray(new WorkspaceReference[0]);
        FunctionAttributes functionAttributes = new FunctionAttributes(template, ps, outputField, works);
        switch(template.scope()) {
            case POINT_AGGREGATE:
                return outputField.isComplexWriter() ? new DrillComplexWriterAggFuncHolder(functionAttributes, initializer) : new DrillAggFuncHolder(functionAttributes, initializer);
            case SIMPLE:
                return outputField.isComplexWriter() ? new DrillComplexWriterFuncHolder(functionAttributes, initializer) : new DrillSimpleFuncHolder(functionAttributes, initializer);
            case HOLISTIC_AGGREGATE:
            case RANGE_AGGREGATE:
            default:
                return failure("Unsupported Function Type.", func);
        }
    } catch (Exception | NoSuchFieldError | AbstractMethodError ex) {
        return failure("Failure while creating function holder.", ex, func);
    }
}