Examples with FileLineReferenceResolver io.crate.operation.reference.file.FileLineReferenceResolver used on opensource projects

Example 1 with FileLineReferenceResolver

use of io.crate.operation.reference.file.FileLineReferenceResolver in project crate by crate.

the class FileReadingCollectorTest method createBatchIterator.

private BatchIterator createBatchIterator(Collection<String> fileUris, String compression, final S3ObjectInputStream s3InputStream) {
    Reference raw = createReference("_raw", DataTypes.STRING);
    InputFactory.Context<LineCollectorExpression<?>> ctx = inputFactory.ctxForRefs(FileLineReferenceResolver::getImplementation);
    List<Input<?>> inputs = Collections.singletonList(ctx.add(raw));
    return FileReadingIterator.newInstance(fileUris, inputs, ctx.expressions(), compression, ImmutableMap.of(LocalFsFileInputFactory.NAME, new LocalFsFileInputFactory(), S3FileInputFactory.NAME, () -> new S3FileInput(new S3ClientHelper() {

        @Override
        protected AmazonS3 initClient(String accessKey, String secretKey) throws IOException {
            AmazonS3 client = mock(AmazonS3Client.class);
            ObjectListing objectListing = mock(ObjectListing.class);
            S3ObjectSummary summary = mock(S3ObjectSummary.class);
            S3Object s3Object = mock(S3Object.class);
            when(client.listObjects(anyString(), anyString())).thenReturn(objectListing);
            when(objectListing.getObjectSummaries()).thenReturn(Arrays.asList(summary));
            when(summary.getKey()).thenReturn("foo");
            when(client.getObject("fakebucket", "foo")).thenReturn(s3Object);
            when(s3Object.getObjectContent()).thenReturn(s3InputStream);
            when(client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
            when(objectListing.isTruncated()).thenReturn(false);
            return client;
        }
    })), false, 1, 0);
}

Example 2 with FileLineReferenceResolver

use of io.crate.operation.reference.file.FileLineReferenceResolver in project crate by crate.

the class FileCollectSource method getCollector.

@Override
public CrateCollector getCollector(CollectPhase collectPhase, BatchConsumer consumer, JobCollectContext jobCollectContext) {
    FileUriCollectPhase fileUriCollectPhase = (FileUriCollectPhase) collectPhase;
    InputFactory.Context<LineCollectorExpression<?>> ctx = inputFactory.ctxForRefs(FileLineReferenceResolver::getImplementation);
    ctx.add(collectPhase.toCollect());
    String[] readers = fileUriCollectPhase.nodeIds().toArray(new String[fileUriCollectPhase.nodeIds().size()]);
    Arrays.sort(readers);
    List<String> fileUris;
    fileUris = targetUriToStringList(fileUriCollectPhase.targetUri());
    BatchIterator fileReadingIterator = FileReadingIterator.newInstance(fileUris, ctx.topLevelInputs(), ctx.expressions(), fileUriCollectPhase.compression(), fileInputFactoryMap, fileUriCollectPhase.sharedStorage(), readers.length, Arrays.binarySearch(readers, clusterService.state().nodes().getLocalNodeId()));
    return BatchIteratorCollectorBridge.newInstance(fileReadingIterator, consumer);
}

Example 3 with FileLineReferenceResolver

use of io.crate.operation.reference.file.FileLineReferenceResolver in project crate by crate.

the class FileReadingIteratorTest method createBatchIterator.

private BatchIterator createBatchIterator(Collection<String> fileUris, String compression) {
    Reference raw = createReference("_raw", DataTypes.STRING);
    InputFactory.Context<LineCollectorExpression<?>> ctx = inputFactory.ctxForRefs(FileLineReferenceResolver::getImplementation);
    List<Input<?>> inputs = Collections.singletonList(ctx.add(raw));
    return FileReadingIterator.newInstance(fileUris, inputs, ctx.expressions(), compression, ImmutableMap.of(LocalFsFileInputFactory.NAME, new LocalFsFileInputFactory()), false, 1, 0);
}