Examples with CsvFileDownloader org.apache.gobblin.source.extractor.filebased.CsvFileDownloader used on opensource projects

Example 1 with CsvFileDownloader

use of org.apache.gobblin.source.extractor.filebased.CsvFileDownloader in project incubator-gobblin by apache.

the class GoogleAnalyticsUnsampledExtractor method prepareUnsampledReport.

/**
 * Create unsampled report in Google drive and add google drive file id into state so that Google drive extractor
 * can extract record from it. Also, update the state to use CsvFileDownloader unless other downloader is defined.
 *
 * It also register closer to delete the file from Google Drive unless explicitly requested to not deleting it.
 * @return documentID of unsampled report in Google drive
 * @throws IOException
 */
@VisibleForTesting
UnsampledReport prepareUnsampledReport(UnsampledReport request, final GoogleDriveFsHelper fsHelper, boolean isDeleteTempReport) throws IOException {
    UnsampledReport createdReport = createUnsampledReports(request);
    final String fileId = createdReport.getDriveDownloadDetails().getDocumentId();
    LOG.info("Temporary unsampled report created in Google Drive: " + fileId);
    if (isDeleteTempReport) {
        closer.register(new Closeable() {

            @Override
            public void close() throws IOException {
                LOG.info("Deleting created temporary unsampled report from Google drive " + fileId);
                fsHelper.deleteFile(fileId);
            }
        });
    } else {
        LOG.warn("Temporary unsampled report will not be deleted as requested. File ID: " + fileId);
    }
    wuState.setProp(SOURCE_FILEBASED_FILES_TO_PULL, fileId);
    if (!wuState.contains(SOURCE_FILEBASED_OPTIONAL_DOWNLOADER_CLASS)) {
        wuState.setProp(SOURCE_FILEBASED_OPTIONAL_DOWNLOADER_CLASS, CsvFileDownloader.class.getName());
    }
    return createdReport;
}