Examples with TableNotFoundException org.apache.hudi.exception.TableNotFoundException used on opensource projects

Example 1 with TableNotFoundException

use of org.apache.hudi.exception.TableNotFoundException in project hudi by apache.

the class TestHoodieRowCreateHandle method testInstantiationFailure.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void testInstantiationFailure(boolean enableMetadataTable) {
    // init config and table
    HoodieWriteConfig cfg = SparkDatasetTestUtils.getConfigBuilder(basePath, timelineServicePort).withPath("/dummypath/abc/").withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(enableMetadataTable).build()).build();
    try {
        HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
        new HoodieRowCreateHandle(table, cfg, " def", UUID.randomUUID().toString(), "001", RANDOM.nextInt(100000), RANDOM.nextLong(), RANDOM.nextLong(), SparkDatasetTestUtils.STRUCT_TYPE);
        fail("Should have thrown exception");
    } catch (HoodieInsertException ioe) {
        // expected without metadata table
        if (enableMetadataTable) {
            fail("Should have thrown TableNotFoundException");
        }
    } catch (TableNotFoundException e) {
        // expected with metadata table
        if (!enableMetadataTable) {
            fail("Should have thrown HoodieInsertException");
        }
    }
}

Example 2 with TableNotFoundException

use of org.apache.hudi.exception.TableNotFoundException in project hudi by apache.

the class InputPathHandler method parseInputPaths.

/**
 * Takes in the original InputPaths and classifies each of them into incremental, snapshot and
 * non-hoodie InputPaths. The logic is as follows:
 * 1. Check if an inputPath starts with the same basePath as any of the metadata basePaths we know
 *    1a. If yes, this belongs to a Hoodie table that we already know about. Simply classify this
 *        as incremental or snapshot - We can get the table name of this inputPath from the
 *        metadata. Then based on the list of incrementalTables, we can classify this inputPath.
 *    1b. If no, this could be a new Hoodie Table we haven't seen yet or a non-Hoodie Input Path.
 *        Try creating the HoodieTableMetadataClient.
 *            - If it succeeds, further classify as incremental on snapshot as described in step
 *              1a above.
 *            - If DatasetNotFoundException/InvalidDatasetException is caught, this is a
 *              non-Hoodie inputPath
 * @param inputPaths - InputPaths from the original jobConf that was passed to HoodieInputFormat
 * @param incrementalTables - List of all incremental tables extracted from the config
 * `hoodie.<table-name>.consume.mode=INCREMENTAL`
 * @throws IOException
 */
private void parseInputPaths(Path[] inputPaths, List<String> incrementalTables) throws IOException {
    for (Path inputPath : inputPaths) {
        boolean basePathKnown = false;
        for (HoodieTableMetaClient metaClient : tableMetaClientMap.values()) {
            if (inputPath.toString().contains(metaClient.getBasePath())) {
                // We already know the base path for this inputPath.
                basePathKnown = true;
                // Check if this is for a snapshot query
                tagAsIncrementalOrSnapshot(inputPath, metaClient, incrementalTables);
                break;
            }
        }
        if (!basePathKnown) {
            // This path is for a table that we don't know about yet.
            HoodieTableMetaClient metaClient;
            try {
                metaClient = getTableMetaClientForBasePathUnchecked(conf, inputPath);
                tableMetaClientMap.put(getIncrementalTable(metaClient), metaClient);
                tagAsIncrementalOrSnapshot(inputPath, metaClient, incrementalTables);
            } catch (TableNotFoundException | InvalidTableException e) {
                // This is a non Hoodie inputPath
                LOG.info("Handling a non-hoodie path " + inputPath);
                nonHoodieInputPaths.add(inputPath);
            }
        }
    }
}

Example 3 with TableNotFoundException

use of org.apache.hudi.exception.TableNotFoundException in project hudi by apache.

the class HoodieJavaStreamingApp method waitTillNCommits.

private void waitTillNCommits(FileSystem fs, int numCommits, int timeoutSecs, int sleepSecsAfterEachRun) throws InterruptedException {
    long beginTime = System.currentTimeMillis();
    long currTime = beginTime;
    long timeoutMsecs = timeoutSecs * 1000;
    while ((currTime - beginTime) < timeoutMsecs) {
        try {
            HoodieTimeline timeline = HoodieDataSourceHelpers.allCompletedCommitsCompactions(fs, tablePath);
            LOG.info("Timeline :" + timeline.getInstants().collect(Collectors.toList()));
            if (timeline.countInstants() >= numCommits) {
                return;
            }
            HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(fs.getConf()).setBasePath(tablePath).setLoadActiveTimelineOnLoad(true).build();
            System.out.println("Instants :" + metaClient.getActiveTimeline().getInstants().collect(Collectors.toList()));
        } catch (TableNotFoundException te) {
            LOG.info("Got table not found exception. Retrying");
        } finally {
            Thread.sleep(sleepSecsAfterEachRun * 1000);
            currTime = System.currentTimeMillis();
        }
    }
    throw new IllegalStateException("Timedout waiting for " + numCommits + " commits to appear in " + tablePath);
}

Example 4 with TableNotFoundException

use of org.apache.hudi.exception.TableNotFoundException in project hudi by apache.

the class HoodieROTablePathFilter method accept.

@Override
public boolean accept(Path path) {
    if (engineContext == null) {
        this.engineContext = new HoodieLocalEngineContext(this.conf.get());
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Checking acceptance for path " + path);
    }
    Path folder = null;
    try {
        if (fs == null) {
            fs = path.getFileSystem(conf.get());
        }
        // Assumes path is a file
        // get the immediate parent.
        folder = path.getParent();
        // Try to use the caches.
        if (nonHoodiePathCache.contains(folder.toString())) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Accepting non-hoodie path from cache: " + path);
            }
            return true;
        }
        if (hoodiePathCache.containsKey(folder.toString())) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("%s Hoodie path checked against cache, accept => %s \n", path, hoodiePathCache.get(folder.toString()).contains(path)));
            }
            return hoodiePathCache.get(folder.toString()).contains(path);
        }
        // Skip all files that are descendants of .hoodie in its path.
        String filePath = path.toString();
        if (filePath.contains("/" + HoodieTableMetaClient.METAFOLDER_NAME + "/") || filePath.endsWith("/" + HoodieTableMetaClient.METAFOLDER_NAME)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("Skipping Hoodie Metadata file  %s \n", filePath));
            }
            return false;
        }
        // Perform actual checking.
        Path baseDir;
        if (HoodiePartitionMetadata.hasPartitionMetadata(fs, folder)) {
            HoodiePartitionMetadata metadata = new HoodiePartitionMetadata(fs, folder);
            metadata.readFromFS();
            baseDir = HoodieHiveUtils.getNthParent(folder, metadata.getPartitionDepth());
        } else {
            baseDir = safeGetParentsParent(folder);
        }
        if (baseDir != null) {
            // Check whether baseDir in nonHoodiePathCache
            if (nonHoodiePathCache.contains(baseDir.toString())) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Accepting non-hoodie path from cache: " + path);
                }
                return true;
            }
            HoodieTableFileSystemView fsView = null;
            try {
                HoodieTableMetaClient metaClient = metaClientCache.get(baseDir.toString());
                if (null == metaClient) {
                    metaClient = HoodieTableMetaClient.builder().setConf(fs.getConf()).setBasePath(baseDir.toString()).setLoadActiveTimelineOnLoad(true).build();
                    metaClientCache.put(baseDir.toString(), metaClient);
                }
                fsView = FileSystemViewManager.createInMemoryFileSystemView(engineContext, metaClient, HoodieInputFormatUtils.buildMetadataConfig(getConf()));
                String partition = FSUtils.getRelativePartitionPath(new Path(metaClient.getBasePath()), folder);
                List<HoodieBaseFile> latestFiles = fsView.getLatestBaseFiles(partition).collect(Collectors.toList());
                // populate the cache
                if (!hoodiePathCache.containsKey(folder.toString())) {
                    hoodiePathCache.put(folder.toString(), new HashSet<>());
                }
                LOG.info("Based on hoodie metadata from base path: " + baseDir.toString() + ", caching " + latestFiles.size() + " files under " + folder);
                for (HoodieBaseFile lfile : latestFiles) {
                    hoodiePathCache.get(folder.toString()).add(new Path(lfile.getPath()));
                }
                // accept the path, if its among the latest files.
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("%s checked after cache population, accept => %s \n", path, hoodiePathCache.get(folder.toString()).contains(path)));
                }
                return hoodiePathCache.get(folder.toString()).contains(path);
            } catch (TableNotFoundException e) {
                // Non-hoodie path, accept it.
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("(1) Caching non-hoodie path under %s with basePath %s \n", folder.toString(), baseDir.toString()));
                }
                nonHoodiePathCache.add(folder.toString());
                nonHoodiePathCache.add(baseDir.toString());
                return true;
            } finally {
                if (fsView != null) {
                    fsView.close();
                }
            }
        } else {
            // files is at < 3 level depth in FS tree, can't be hoodie dataset
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("(2) Caching non-hoodie path under %s \n", folder.toString()));
            }
            nonHoodiePathCache.add(folder.toString());
            return true;
        }
    } catch (Exception e) {
        String msg = "Error checking path :" + path + ", under folder: " + folder;
        LOG.error(msg, e);
        throw new HoodieException(msg, e);
    }
}