Examples with FileMetadataTransformResponseModifier com.thinkbiganalytics.spark.rest.filemetadata.FileMetadataTransformResponseModifier used on opensource projects

Example 1 with FileMetadataTransformResponseModifier

use of com.thinkbiganalytics.spark.rest.filemetadata.FileMetadataTransformResponseModifier in project kylo by Teradata.

the class FileMetadataTaskService method findFileMetadataSchemas.

/**
 * Group the files by their respective mime type
 * For each mime type that spark can process create a task to determine the header information
 */
public void findFileMetadataSchemas(ModifiedTransformResponse<FileMetadataResponse> modifiedTransformResponse, FileMetadataTransformResponseModifier resultModifier) {
    FileMetadataCompletionTask result = new FileMetadataCompletionTask(modifiedTransformResponse, resultModifier);
    metadataResultCache.put(result.getTableId(), result);
    Map<String, List<com.thinkbiganalytics.spark.rest.model.FileMetadataResponse.ParsedFileMetadata>> mimeTypeGroup = resultModifier.groupByMimeType();
    List<String> mimeTypes = Lists.newArrayList(mimeTypeGroup.keySet());
    mimeTypes.removeIf(type -> !PARSABLE_MIME_TYPES.contains(type));
    List<SparkShellScriptRunner> tasks = new ArrayList<>();
    for (String mimeType : mimeTypes) {
        List<com.thinkbiganalytics.spark.rest.model.FileMetadataResponse.ParsedFileMetadata> data = mimeTypeGroup.get(mimeType);
        if (mimeType == "application/xml") {
            // need to group by rowtag
            Map<String, List<com.thinkbiganalytics.spark.rest.model.FileMetadataResponse.ParsedFileMetadata>> rowTags = data.stream().collect(Collectors.groupingBy(row -> row.getRowTag()));
            for (Map.Entry<String, List<com.thinkbiganalytics.spark.rest.model.FileMetadataResponse.ParsedFileMetadata>> rows : rowTags.entrySet()) {
                List<String> files = rows.getValue().stream().map(r -> r.getFilePath()).collect(Collectors.toList());
                SparkShellScriptRunner shellScriptRunner = new SparkShellScriptRunner(sparkShellUserProcessService, restClient, getUsername(), FileMetadataSchemaScriptBuilder.getSparkScript(mimeType, rows.getKey(), files), mimeType);
                tasks.add(shellScriptRunner);
                result.addTask(shellScriptRunner, rows.getValue());
            }
        } else {
            List<String> files = data.stream().map(r -> r.getFilePath()).collect(Collectors.toList());
            SparkShellScriptRunner shellScriptRunner = new SparkShellScriptRunner(sparkShellUserProcessService, restClient, getUsername(), FileMetadataSchemaScriptBuilder.getSparkScript(mimeType, null, files), mimeType);
            tasks.add(shellScriptRunner);
            result.addTask(shellScriptRunner, data);
        }
    }
    submitTasks(result, tasks);
}

Example 2 with FileMetadataTransformResponseModifier

use of com.thinkbiganalytics.spark.rest.filemetadata.FileMetadataTransformResponseModifier in project kylo by Teradata.

the class FileMetadataTest method testChainedResult.

@Test
public void testChainedResult() {
    setup();
    TransformResponse initialResponse = new TransformResponse();
    initialResponse.setStatus(TransformResponse.Status.SUCCESS);
    TransformQueryResult result = new TransformQueryResult();
    List<QueryResultColumn> columnList = new ArrayList<>();
    columnList.add(newColumn("mimeType"));
    columnList.add(newColumn("delimiter"));
    columnList.add(newColumn("headerCount"));
    columnList.add(newColumn("resource"));
    columnList.add(newColumn("encoding"));
    result.setColumns(columnList);
    List<List<Object>> rows = new ArrayList<>();
    rows.add(newtRow("application/parquet", "file://my/parquet001.parquet"));
    rows.add(newtRow("application/parquet", "file://my/parquet002.parquet"));
    rows.add(newtRow("application/parquet", "file://my/parquet003.parquet"));
    rows.add(newtRow("application/avro", "file://my/avro001.avro"));
    rows.add(newtRow("application/avro", "file://my/avro002.avro"));
    rows.add(newtRow("text/csv", "file://my/test001.csv"));
    rows.add(newtRow("text/csv", "file://my/test002.csv"));
    result.setRows(rows);
    initialResponse.setResults(result);
    initialResponse.setTable(UUID.randomUUID().toString());
    FileMetadataTransformResponseModifier fileMetadataResult = new FileMetadataTransformResponseModifier(trackerService);
    ModifiedTransformResponse<FileMetadataResponse> m = fileMetadataResult.modify(initialResponse);
    FileMetadataResponse response = m.getResults();
    int retryCount = 0;
    long start = System.currentTimeMillis();
    boolean process = response == null;
    while (process) {
        Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS);
        response = m.getResults();
        if (response != null) {
            process = false;
        }
        retryCount += 1;
        if (retryCount > 40) {
            process = false;
        }
    }
    long stop = System.currentTimeMillis();
    log.info("Time to get chained response {} ms, Retry Attempts: {}", (stop - start), retryCount);
    Assert.assertEquals(3, response.getDatasets().size());
    Assert.assertEquals(2, response.getDatasets().get("file://my/test001.csv").getFiles().size());
    Assert.assertEquals(3, response.getDatasets().get("file://my/parquet001.parquet").getFiles().size());
    Assert.assertEquals(2, response.getDatasets().get("file://my/avro001.avro").getFiles().size());
}

Example 3 with FileMetadataTransformResponseModifier

use of com.thinkbiganalytics.spark.rest.filemetadata.FileMetadataTransformResponseModifier in project kylo by Teradata.

the class SparkShellProxyController method fileMetadata.

@POST
@Path(FILE_METADATA)
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("returns filemetadata based upon the list of file paths in the dataset.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the status of the file-metadata job.", response = TransformResponse.class), @ApiResponse(code = 400, message = "The requested data source does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "There was a problem processing the data.", response = RestResponseStatus.class) })
public Response fileMetadata(com.thinkbiganalytics.kylo.catalog.rest.model.DataSet dataSet) {
    TransformRequest request = new TransformRequest();
    DataSet decrypted = catalogModelTransform.decryptOptions(dataSet);
    request.setScript(FileMetadataScalaScriptGenerator.getScript(DataSetUtil.getPaths(decrypted).orElseGet(Collections::emptyList), DataSetUtil.mergeTemplates(decrypted).getOptions()));
    final SparkShellProcess process = getSparkShellProcess();
    return getModifiedTransformResponse(() -> Optional.of(restClient.transform(process, request)), new FileMetadataTransformResponseModifier(fileMetadataTrackerService));
}