diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java index c451db21dfa95..c2cbe30cd0b5a 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java @@ -39,7 +39,6 @@ import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.storage.HoodieStorage; -import org.apache.hudi.storage.HoodieStorageUtils; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; @@ -116,7 +115,7 @@ public String showArchivedCommits( List allStats = new ArrayList<>(); for (StoragePathInfo pathInfo : pathInfoList) { // read the archived file - try (Reader reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(pathInfo.getPath()), + try (Reader reader = HoodieLogFormat.newReader(metaClient, new HoodieLogFile(pathInfo.getPath()), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) { List readRecords = new ArrayList<>(); // read the avro blocks @@ -182,15 +181,14 @@ public String showCommits( System.out.println("===============> Showing only " + limit + " archived commits <==============="); HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient(); - StoragePath basePath = metaClient.getBasePath(); StoragePath archivePath = new StoragePath(metaClient.getArchivePath(), ".commits_.archive*"); - List pathInfoList = - HoodieStorageUtils.getStorage(basePath, HoodieCLI.conf).globEntries(archivePath); + HoodieStorage storage = metaClient.getStorage(); + List pathInfoList = storage.globEntries(archivePath); List allCommits = new ArrayList<>(); for (StoragePathInfo pathInfo : pathInfoList) { // read the archived file - try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(HoodieStorageUtils.getStorage(basePath, HoodieCLI.conf), + try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(metaClient, new HoodieLogFile(pathInfo.getPath()), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) { List readRecords = new ArrayList<>(); // read the avro blocks diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java index 83bdbbdb9c5c0..c10c3413125f0 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java @@ -126,12 +126,13 @@ private int copyArchivedInstants(List pathInfoList, int limit, String localFolder) throws Exception { int copyCount = 0; - HoodieStorage storage = HoodieStorageUtils.getStorage( - HoodieCLI.getTableMetaClient().getBasePath(), HoodieCLI.conf); + HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient(); + HoodieStorage storage = metaClient.getStorage(); for (StoragePathInfo pathInfo : pathInfoList) { // read the archived file - try (Reader reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(pathInfo.getPath()), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) { + try (Reader reader = HoodieLogFormat.newReader(metaClient, new HoodieLogFile(pathInfo.getPath()), + HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) { // read the avro blocks while (reader.hasNext() && copyCount++ < limit) { diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java index d6d2546e2cb11..9f1ed49938097 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java @@ -94,7 +94,8 @@ public String showLogFileCommits( defaultValue = "false") final boolean headerOnly) throws IOException { - HoodieStorage storage = HoodieCLI.getTableMetaClient().getStorage(); + HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient(); + HoodieStorage storage = metaClient.getStorage(); List logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder( storage, new StoragePath(logFilePathPattern)).stream() .map(status -> status.getPath().toString()).collect(Collectors.toList()); @@ -115,8 +116,8 @@ storage, new StoragePath(logFilePathPattern)).stream() } else { fileName = path.getName(); } - HoodieSchema writerSchema = TableSchemaResolver.readSchemaFromLogFile(storage, path); - try (Reader reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(path), writerSchema)) { + HoodieSchema writerSchema = TableSchemaResolver.readSchemaFromLogFile(metaClient, path); + try (Reader reader = HoodieLogFormat.newReader(metaClient, new HoodieLogFile(path), writerSchema)) { // read the avro blocks while (reader.hasNext()) { @@ -221,7 +222,7 @@ storage, new StoragePath(logFilePathPattern)).stream() // get schema from last log file for (int i = logFilePaths.size() - 1; i >= 0; i--) { HoodieSchema schema = TableSchemaResolver.readSchemaFromLogFile( - storage, new StoragePath(logFilePaths.get(i))); + client, new StoragePath(logFilePaths.get(i))); if (schema != null) { readerSchema = schema; break; @@ -264,9 +265,9 @@ storage, new StoragePath(logFilePathPattern)).stream() } else { for (String logFile : logFilePaths) { HoodieSchema writerSchema = TableSchemaResolver.readSchemaFromLogFile( - client.getStorage(), new StoragePath(logFile)); + client, new StoragePath(logFile)); try (HoodieLogFormat.Reader reader = - HoodieLogFormat.newReader(storage, new HoodieLogFile(new StoragePath(logFile)), writerSchema)) { + HoodieLogFormat.newReader(client, new HoodieLogFile(new StoragePath(logFile)), writerSchema)) { // read the avro blocks while (reader.hasNext()) { HoodieLogBlock n = reader.next(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LegacyArchivedMetaEntryReader.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LegacyArchivedMetaEntryReader.java index 9feb958aa38cf..833987c45f2af 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LegacyArchivedMetaEntryReader.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LegacyArchivedMetaEntryReader.java @@ -269,7 +269,7 @@ public boolean hasNext() { StoragePathInfo pathInfo = pathInfoIterator.next(); try { reader = HoodieLogFormat.newReader( - metaClient.getStorage(), + metaClient, new HoodieLogFile(pathInfo.getPath()), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema())); } catch (IOException ioe) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 7918e0f7b9a04..7aadac3d34079 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -153,7 +153,8 @@ public class HoodieWriteConfig extends HoodieConfig { .withValidValues( String.valueOf(HoodieTableVersion.SIX.versionCode()), String.valueOf(HoodieTableVersion.EIGHT.versionCode()), - String.valueOf(HoodieTableVersion.NINE.versionCode()) + String.valueOf(HoodieTableVersion.NINE.versionCode()), + String.valueOf(HoodieTableVersion.TEN.versionCode()) ) .sinceVersion("1.0.0") .withDocumentation("The table version this writer is storing the table in. This should match the current table version."); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/AppendHandleFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/AppendHandleFactory.java index 9df089a4914fe..25d49890366dd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/AppendHandleFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/AppendHandleFactory.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.util.CommonClientUtils; import java.util.Iterator; @@ -33,7 +34,7 @@ public HoodieAppendHandle create(final HoodieWriteConfig hoodieConfi final String fileIdPrefix, final TaskContextSupplier sparkTaskContextSupplier) { String fileId = getNextFileId(fileIdPrefix); - if (hoodieTable.getMetaClient().getTableConfig().isLSMTreeStorageLayout()) { + if (CommonClientUtils.shouldWriteNativeLogs(hoodieConfig, hoodieTable.getMetaClient().getTableConfig())) { return new HoodieNativeLogAppendHandle<>(hoodieConfig, commitTime, hoodieTable, partitionPath, fileId, sparkTaskContextSupplier); } @@ -45,7 +46,7 @@ public HoodieAppendHandle create(final HoodieWriteConfig hoodieConfi final HoodieTable hoodieTable, final String partitionPath, final String fileId, final Iterator> recordItr, final TaskContextSupplier sparkTaskContextSupplier) { - if (hoodieTable.getMetaClient().getTableConfig().isLSMTreeStorageLayout()) { + if (CommonClientUtils.shouldWriteNativeLogs(hoodieConfig, hoodieTable.getMetaClient().getTableConfig())) { return new HoodieNativeLogAppendHandle<>(hoodieConfig, commitTime, hoodieTable, partitionPath, fileId, recordItr, sparkTaskContextSupplier); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java index 8c92660af6ef1..393e69ef2539d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java @@ -42,6 +42,7 @@ import org.apache.hudi.common.schema.internal.InternalSchema; import org.apache.hudi.common.schema.internal.utils.AvroSchemaEvolutionUtils; import org.apache.hudi.common.schema.internal.utils.SerDeHelper; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.cdc.HoodieCDCUtils; import org.apache.hudi.common.table.read.BaseFileUpdateCallback; import org.apache.hudi.common.table.read.BufferedRecord; @@ -55,13 +56,13 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.io.cdc.HoodieCDCLogWriter; -import org.apache.hudi.io.cdc.HoodieCDCLogWriterFactory; import org.apache.hudi.io.cdc.HoodieCDCLogger; import org.apache.hudi.io.cdc.HoodieNativeCDCLogger; import org.apache.hudi.keygen.BaseKeyGenerator; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.compact.strategy.CompactionStrategy; +import org.apache.hudi.util.CommonClientUtils; import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; @@ -141,7 +142,7 @@ public FileGroupReaderBasedMergeHandle(HoodieWriteConfig config, String instantT TypedProperties properties = config.getProps(); properties.putAll(hoodieTable.getMetaClient().getTableConfig().getProps()); this.maxInstantTime = instantTime; - initRecordTypeAndCdcLogger(hoodieTable.getConfig().getRecordMerger().getRecordType()); + initRecordType(hoodieTable.getConfig().getRecordMerger().getRecordType()); this.props = TypedProperties.copy(config.getProps()); this.incomingRecordsItr = recordItr; } @@ -168,28 +169,24 @@ public FileGroupReaderBasedMergeHandle(HoodieWriteConfig config, String instantT this.keyToNewRecords = Collections.emptyMap(); this.readerContext = readerContext; this.compactionOperation = Option.of(compactionOperation); - initRecordTypeAndCdcLogger(enginRecordType); + initRecordType(enginRecordType); init(compactionOperation, this.partitionPath); this.props = TypedProperties.copy(config.getProps()); this.incomingRecordsItr = null; } - private void initRecordTypeAndCdcLogger(HoodieRecord.HoodieRecordType enginRecordType) { + private void initRecordType(HoodieRecord.HoodieRecordType enginRecordType) { // If the table is a metadata table or the base file is an HFile, we use AVRO record type, otherwise we use the engine record type. this.recordType = hoodieTable.isMetadataTable() || HFILE.getFileExtension().equals(hoodieTable.getBaseFileExtension()) ? HoodieRecord.HoodieRecordType.AVRO : enginRecordType; - if (hoodieTable.getMetaClient().getTableConfig().isCDCEnabled()) { - this.cdcLogger = Option.of(createCDCLogWriter()); - } else { - this.cdcLogger = Option.empty(); - } } private HoodieCDCLogWriter createCDCLogWriter() { - if (HoodieCDCLogWriterFactory.shouldWriteNativeCDCLogs(hoodieTable.getMetaClient().getTableConfig())) { + HoodieTableConfig tableConfig = hoodieTable.getMetaClient().getTableConfig(); + if (CommonClientUtils.shouldWriteNativeLogs(config, tableConfig)) { return new HoodieNativeCDCLogger( instantTime, config, - hoodieTable.getMetaClient().getTableConfig(), + tableConfig, partitionPath, storage, getWriterSchema(), @@ -204,7 +201,7 @@ private HoodieCDCLogWriter createCDCLogWriter() { return new HoodieCDCLogger( instantTime, config, - hoodieTable.getMetaClient().getTableConfig(), + tableConfig, partitionPath, storage, getWriterSchema(), @@ -395,6 +392,7 @@ public List close() { protected Option> createCallback() { List> callbacks = new ArrayList<>(); // Handle CDC workflow. + Option> cdcLogger = getOrCreateCDCLogWriter(); if (cdcLogger.isPresent()) { HoodieCDCLogWriter logger = cdcLogger.get(); if (logger instanceof HoodieNativeCDCLogger) { @@ -428,6 +426,18 @@ protected Option> createCallback() { return callbacks.isEmpty() ? Option.empty() : Option.of(CompositeCallback.of(callbacks)); } + private Option> getOrCreateCDCLogWriter() { + if (cdcLogger != null) { + return cdcLogger; + } + if (!hoodieTable.getMetaClient().getTableConfig().isCDCEnabled()) { + this.cdcLogger = Option.empty(); + } else { + cdcLogger = Option.of(createCDCLogWriter()); + } + return cdcLogger; + } + private static class CDCCallback implements BaseFileUpdateCallback { private final HoodieCDCLogWriter cdcLogger; private final RecordContext recordContext; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedNativeLogAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedNativeLogAppendHandle.java index f68a57eb991ae..c30a5f8bca1ef 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedNativeLogAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedNativeLogAppendHandle.java @@ -45,7 +45,6 @@ import javax.annotation.concurrent.NotThreadSafe; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.stream.Stream; @@ -111,24 +110,36 @@ public void doAppend() { @Override public List close() { try { - super.close(); - writeStatus.getStat().setPartitionPath(operation.getPartitionPath()); - writeStatus.getStat().setTotalLogReadTimeMs(readStats.getTotalLogReadTimeMs()); - writeStatus.getStat().setTotalUpdatedRecordsCompacted(readStats.getTotalUpdatedRecordsCompacted()); - writeStatus.getStat().setTotalLogFilesCompacted(readStats.getTotalLogFilesCompacted()); - writeStatus.getStat().setTotalLogRecords(readStats.getTotalLogRecords()); - writeStatus.getStat().setTotalLogBlocks(readStats.getTotalLogBlocks()); - writeStatus.getStat().setTotalCorruptLogBlock(readStats.getTotalCorruptLogBlock()); - writeStatus.getStat().setTotalRollbackBlocks(readStats.getTotalRollbackBlocks()); - writeStatus.getStat().setTotalLogSizeCompacted(readStats.getTotalLogSizeCompacted()); - - if (writeStatus.getStat().getRuntimeStats() != null) { - writeStatus.getStat().getRuntimeStats().setTotalScanTime(readStats.getTotalLogReadTimeMs()); + List writeStatuses = super.close(); + // Native log compaction can produce more than one physical output file, for example a data log + // and a delete log, so keep all write statuses and annotate each output file with compaction identity. + for (WriteStatus status : writeStatuses) { + status.getStat().setPartitionPath(operation.getPartitionPath()); + status.getStat().setPrevCommit(operation.getBaseInstantTime()); + } + // The read stats describe the source file group scanned for this compaction, not each native + // output file. Set them once so commit-level compaction metrics are not double-counted. + if (!writeStatuses.isEmpty()) { + updateCompactionReadStats(writeStatuses.get(0)); } - writeStatus.getStat().setPrevCommit(operation.getBaseInstantTime()); - return Collections.singletonList(writeStatus); + return writeStatuses; } catch (Exception e) { throw new HoodieUpsertException("Failed to close " + this.getClass().getSimpleName(), e); } } + + private void updateCompactionReadStats(WriteStatus status) { + status.getStat().setTotalLogReadTimeMs(readStats.getTotalLogReadTimeMs()); + status.getStat().setTotalUpdatedRecordsCompacted(readStats.getTotalUpdatedRecordsCompacted()); + status.getStat().setTotalLogFilesCompacted(readStats.getTotalLogFilesCompacted()); + status.getStat().setTotalLogRecords(readStats.getTotalLogRecords()); + status.getStat().setTotalLogBlocks(readStats.getTotalLogBlocks()); + status.getStat().setTotalCorruptLogBlock(readStats.getTotalCorruptLogBlock()); + status.getStat().setTotalRollbackBlocks(readStats.getTotalRollbackBlocks()); + status.getStat().setTotalLogSizeCompacted(readStats.getTotalLogSizeCompacted()); + + if (status.getStat().getRuntimeStats() != null) { + status.getStat().getRuntimeStats().setTotalScanTime(readStats.getTotalLogReadTimeMs()); + } + } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java index ef3b92849dc09..53bec47dca551 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java @@ -44,6 +44,8 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.ExceptionUtil; +import org.apache.hudi.exception.HoodieEarlyConflictDetectionException; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.storage.StoragePath; @@ -379,7 +381,7 @@ protected boolean writeRecord(HoodieRecord hoodieRecord) { return true; } catch (Exception e) { log.error("Error writing record {}", hoodieRecord, e); - if (!config.getIgnoreWriteFailed()) { + if (!config.getIgnoreWriteFailed() || ExceptionUtil.isCausedBy(e, HoodieEarlyConflictDetectionException.class)) { throw new HoodieException(e.getMessage(), e); } writeStatus.markFailure(hoodieRecord, e, recordMetadata); @@ -439,24 +441,14 @@ protected Map getUpdatedHeader(MapSubclasses call this after they flush buffered inline blocks or native writer state. The method updates - * write stats, records log-file membership, collects optional column stats, resets per-flush counters, and - * restarts the runtime timer for the next append. - */ - protected void processAppendResult(AppendResult result) { + protected void processAppendResult(AppendResult result, long elapsedTime) { HoodieDeltaWriteStat stat = (HoodieDeltaWriteStat) this.writeStatus.getStat(); - updateWriteStatus(result, stat); + updateWriteStatus(result, stat, elapsedTime); stat = (HoodieDeltaWriteStat) this.writeStatus.getStat(); - updateLogFiles(stat); collectColumnStats(stat); - resetWriteCounts(); assert stat.getRuntimeStats() != null; log.info("AppendHandle for partitionPath {} filePath {}, took {} ms.", partitionPath, stat.getPath(), stat.getRuntimeStats().getTotalUpsertTime()); - timer.startTimer(); } /** @@ -465,23 +457,23 @@ protected void processAppendResult(AppendResult result) { *

A single handle can write multiple physical log files because of rollover. Appends to the same log file * accumulate into the current stat; appends to a new log file create a new write status. */ - protected void updateWriteStatus(AppendResult result, HoodieDeltaWriteStat stat) { + protected void updateWriteStatus(AppendResult result, HoodieDeltaWriteStat stat, long elapsedTime) { if (stat.getPath() == null) { updateWriteStat(stat, result); updateWriteCounts(stat, result); - updateRuntimeStats(stat); + updateRuntimeStats(stat, elapsedTime); statuses.add(this.writeStatus); } else if (stat.getPath().endsWith(result.logFile().getFileName())) { stat.setLogOffset(Math.min(stat.getLogOffset(), result.offset())); stat.setFileSizeInBytes(stat.getFileSizeInBytes() + result.size()); accumulateWriteCounts(stat, result); - accumulateRuntimeStats(stat); + accumulateRuntimeStats(stat, elapsedTime); } else { initNewStatus(); stat = (HoodieDeltaWriteStat) this.writeStatus.getStat(); updateWriteStat(stat, result); updateWriteCounts(stat, result); - updateRuntimeStats(stat); + updateRuntimeStats(stat, elapsedTime); statuses.add(this.writeStatus); } } @@ -506,15 +498,6 @@ protected void resetWriteCounts() { recordsDeleted = 0; } - /** - * Allows subclasses to add extra log file names to the current stat after a flush. - * - *

Inline log appends usually have a single {@link AppendResult}. Native log appends can flush both data and - * delete files together, so they use this hook to record every physical file written by the flush. - */ - protected void updateLogFiles(HoodieDeltaWriteStat stat) { - } - /** * Allows subclasses to collect column stats after a flush. * @@ -534,7 +517,7 @@ private void updateWriteStat(HoodieDeltaWriteStat stat, AppendResult result) { stat.setFileSizeInBytes(result.size()); } - private void updateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) { + protected void updateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) { stat.setNumWrites(recordsWritten); stat.setNumUpdateWrites(updatedRecordsWritten); stat.setNumInserts(insertRecordsWritten); @@ -542,7 +525,7 @@ private void updateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) { stat.setTotalWriteBytes(result.size()); } - private void accumulateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) { + protected void accumulateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) { stat.setNumWrites(stat.getNumWrites() + recordsWritten); stat.setNumUpdateWrites(stat.getNumUpdateWrites() + updatedRecordsWritten); stat.setNumInserts(stat.getNumInserts() + insertRecordsWritten); @@ -550,16 +533,16 @@ private void accumulateWriteCounts(HoodieDeltaWriteStat stat, AppendResult resul stat.setTotalWriteBytes(stat.getTotalWriteBytes() + result.size()); } - private void updateRuntimeStats(HoodieDeltaWriteStat stat) { + private void updateRuntimeStats(HoodieDeltaWriteStat stat, long elapsedTime) { RuntimeStats runtimeStats = new RuntimeStats(); - runtimeStats.setTotalUpsertTime(timer.endTimer()); + runtimeStats.setTotalUpsertTime(elapsedTime); stat.setRuntimeStats(runtimeStats); } - private void accumulateRuntimeStats(HoodieDeltaWriteStat stat) { + private void accumulateRuntimeStats(HoodieDeltaWriteStat stat, long elapsedTime) { RuntimeStats runtimeStats = stat.getRuntimeStats(); assert runtimeStats != null; - runtimeStats.setTotalUpsertTime(runtimeStats.getTotalUpsertTime() + timer.endTimer()); + runtimeStats.setTotalUpsertTime(runtimeStats.getTotalUpsertTime() + elapsedTime); } private String makeFilePath(HoodieLogFile logFile) { @@ -584,7 +567,7 @@ private String makeFilePath(HoodieLogFile logFile) { protected abstract void writeDelete(HoodieSchema schema, HoodieRecord hoodieRecord) throws IOException; /** - * Flushes any pending data/delete records and calls {@link #processAppendResult(AppendResult)} for each append result. + * Flushes any pending data/delete records and updates write stats for the flushed append result. */ protected abstract void flushAppend(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieInlineLogAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieInlineLogAppendHandle.java index 80020a799089f..ea32b29674ab7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieInlineLogAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieInlineLogAppendHandle.java @@ -164,6 +164,16 @@ protected void flushAppend() { numberOfRecords = 0; } + /** + * Finalizes accounting for one flushed inline-log append. + */ + protected void processAppendResult(AppendResult result) { + long elapsedTime = timer.endTimer(); + processAppendResult(result, elapsedTime); + resetWriteCounts(); + timer.startTimer(); + } + @Override protected void collectColumnStats(HoodieDeltaWriteStat stat) { if (config.isMetadataColumnStatsIndexEnabled()) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieNativeLogAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieNativeLogAppendHandle.java index ecf6dc44c7edd..5950842d4045c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieNativeLogAppendHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieNativeLogAppendHandle.java @@ -26,7 +26,7 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.log.AppendResult; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import org.apache.hudi.common.util.ConfigUtils; +import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ParquetUtils; import org.apache.hudi.config.HoodieWriteConfig; @@ -38,16 +38,17 @@ import org.apache.hudi.stats.HoodieColumnRangeMetadata; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.util.CommonClientUtils; import org.apache.hudi.util.Lazy; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -84,12 +85,15 @@ private HoodieNativeLogAppendHandle(HoodieWriteConfig config, String instantTime TaskContextSupplier taskContextSupplier, boolean preserveMetadata, Map header) { super(config, instantTime, hoodieTable, partitionPath, fileId, recordItr, taskContextSupplier, preserveMetadata); + CommonClientUtils.validateIndexSupportForNativeLogFormat(config, hoodieTable.getBaseFileFormat()); this.header.putAll(header); } @Override protected void createLogWriterForAppend(String instantTime, Option fileSliceOpt) { try { + List orderingFields = HoodieRecordUtils.getOrderingFieldNames( + hoodieTable.getMetaClient().getTableConfig().getRecordMergeMode(), hoodieTable.getMetaClient()); this.writer = new HoodieNativeLogFormatWriter( storage.getDefaultBufferSize(), storage, @@ -106,7 +110,8 @@ protected void createLogWriterForAppend(String instantTime, Option fi writeSchemaWithMetaFields, taskContextSupplier, hoodieTable.getReaderContextFactoryForWrite().getContext().getRecordContext(), - Arrays.stream(ConfigUtils.getOrderingFields(recordProperties)).collect(Collectors.toList())); + orderingFields, + baseFileInstantTimeOfPositions); } catch (IOException e) { throw new HoodieException("Creating native log writer with fileId: " + fileId + ", " + "delta commit time: " + instantTime + " error", e); @@ -156,7 +161,8 @@ protected void flushAppend() { header.put(HeaderMetadataType.INSTANT_TIME, instantTime); header.put(HeaderMetadataType.SCHEMA, writeSchemaWithMetaFields.toString()); if (writer != null && writer.hasPendingWrites()) { - processAppendResult(writer.flushAppend(getUpdatedHeader(header))); + writer.flushAppend(getUpdatedHeader(header)); + processAppendResults(writer.getLastAppendResults()); } } catch (IOException e) { throw new HoodieAppendException("Failed while flushing records to native log for fileId " + fileId, e); @@ -171,6 +177,26 @@ protected void closeLogWriter() { } } + /** + * Finalizes accounting for all physical native log files produced by one flush. + * + *

The native writer returns append results in file-write order: data log first and delete log second when both + * files are produced. Delete-only flushes contain a single delete result at index 0. + */ + private void processAppendResults(List results) { + long elapsedTime = timer.endTimer(); + try { + for (int i = 0; i < results.size(); i++) { + processAppendResult(results.get(i), i == 0 ? elapsedTime : 0L); + } + if (!results.isEmpty()) { + resetWriteCounts(); + } + } finally { + timer.startTimer(); + } + } + protected StoragePath getLogFilePath() { return writer.getLogFile().getPath(); } @@ -181,28 +207,54 @@ public boolean canWrite(HoodieRecord record) { } @Override - protected void updateLogFiles(HoodieDeltaWriteStat stat) { - for (AppendResult appendResult : writer.getLastAppendResults()) { - if (!stat.getLogFiles().contains(appendResult.logFile().getFileName())) { - stat.addLogFiles(appendResult.logFile().getFileName()); - } + protected void collectColumnStats(HoodieDeltaWriteStat stat) { + if (!config.isMetadataColumnStatsIndexEnabled()) { + return; + } + + HoodieIndexVersion indexVersion = HoodieTableMetadataUtil.existingIndexVersionOrDefault(PARTITION_NAME_COLUMN_STATS, hoodieTable.getMetaClient()); + Set columnsToIndexSet = new HashSet<>(HoodieTableMetadataUtil + .getColumnsToIndex(hoodieTable.getMetaClient().getTableConfig(), + config.getMetadataConfig(), Lazy.eagerly(Option.of(writeSchemaWithMetaFields)), + Option.of(recordMerger.getRecordType()), indexVersion).keySet()); + + if (FSUtils.isNativeDeleteLogFile(new StoragePath(stat.getPath()).getName())) { + // Native delete logs do not contain data values, so there is no min/max range to collect. Still, publishing + // empty column stats is meaningful for query-side column-stats pruning: it marks the delete log as indexed but + // empty. Without these records, the query path treats the log as an un-indexed file and keeps the file slice as a + // fallback candidate, which is not equivalent to the legacy inline-log behavior for delete-only blocks. + stat.putRecordsStats(columnsToIndexSet.stream() + .collect(Collectors.toMap( + column -> column, + column -> HoodieColumnRangeMetadata.createEmpty(stat.getPath(), column, indexVersion)))); + return; } - } - @Override - protected void collectColumnStats(HoodieDeltaWriteStat stat) { Option dataFileFormatMetadata = writer.getLastDataFileFormatMetadata(); - if (config.isMetadataColumnStatsIndexEnabled() && dataFileFormatMetadata.isPresent()) { - HoodieIndexVersion indexVersion = HoodieTableMetadataUtil.existingIndexVersionOrDefault(PARTITION_NAME_COLUMN_STATS, hoodieTable.getMetaClient()); - Set columnsToIndexSet = new HashSet<>(HoodieTableMetadataUtil - .getColumnsToIndex(hoodieTable.getMetaClient().getTableConfig(), - config.getMetadataConfig(), Lazy.eagerly(Option.of(writeSchemaWithMetaFields)), - Option.of(recordMerger.getRecordType()), indexVersion).keySet()); + if (dataFileFormatMetadata.isPresent()) { stat.putRecordsStats(collectNativeLogColumnRangeMetadata( stat.getPath(), dataFileFormatMetadata.get(), columnsToIndexSet, indexVersion)); } } + @Override + protected void updateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) { + if (FSUtils.isNativeDeleteLogFile(result.logFile().getFileName())) { + stat.setNumDeletes(recordsDeleted); + } else { + stat.setNumWrites(recordsWritten); + stat.setNumUpdateWrites(updatedRecordsWritten); + stat.setNumInserts(insertRecordsWritten); + } + stat.setTotalWriteBytes(result.size()); + } + + @Override + protected void accumulateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) { + throw new HoodieAppendException("Native format log append should not accumulate write counts for " + + result.logFile().getPath()); + } + private Map> collectNativeLogColumnRangeMetadata( String filePath, Object metadata, diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieAvroNativeCDCLogger.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieAvroNativeCDCLogger.java index 9a9362f716f0a..999a1df9bd88a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieAvroNativeCDCLogger.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieAvroNativeCDCLogger.java @@ -20,8 +20,6 @@ import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.engine.TaskContextSupplier; -import org.apache.hudi.common.model.HoodieAvroIndexedRecord; -import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaCache; @@ -54,12 +52,11 @@ public class HoodieAvroNativeCDCLogger implements HoodieCDCLogWriter { private final String commitTime; - private final String partitionPath; private final HoodieSchema dataSchema; private final HoodieSchema cdcSchema; private final HoodieCDCSupplementalLoggingMode cdcSupplementalLoggingMode; private final CDCTransformer transformer; - private final HoodieNativeCDCFileWriter nativeCDCFileWriter; + private final HoodieNativeCDCFileWriter nativeCDCFileWriter; private PendingCDCRecord pendingRecord; public HoodieAvroNativeCDCLogger( @@ -75,12 +72,11 @@ public HoodieAvroNativeCDCLogger( LogFileCreationCallback fileCreationCallback, TaskContextSupplier taskContextSupplier) { this.commitTime = commitTime; - this.partitionPath = partitionPath; this.dataSchema = HoodieSchemaCache.intern(HoodieSchemaUtils.removeMetadataFields(schema)); this.cdcSupplementalLoggingMode = tableConfig.cdcSupplementalLoggingMode(); this.cdcSchema = HoodieCDCUtils.schemaBySupplementalLoggingMode(cdcSupplementalLoggingMode, dataSchema); this.transformer = getTransformer(); - this.nativeCDCFileWriter = new HoodieNativeCDCFileWriter( + this.nativeCDCFileWriter = new HoodieNativeCDCFileWriter<>( commitTime, partitionPath, storage, @@ -139,9 +135,7 @@ private void flushPendingRecord() { return; } try { - nativeCDCFileWriter.write( - pendingRecord.recordKey, - new HoodieAvroIndexedRecord(new HoodieKey(pendingRecord.recordKey, partitionPath), pendingRecord.record)); + nativeCDCFileWriter.write(pendingRecord.recordKey, pendingRecord.record); pendingRecord = null; } catch (IOException e) { throw new HoodieException("Failed to write the cdc data to native cdc log file", e); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieCDCLogWriterFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieCDCLogWriterFactory.java index a5aed0062347f..9ba1f30e97244 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieCDCLogWriterFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieCDCLogWriterFactory.java @@ -28,6 +28,7 @@ import org.apache.hudi.io.IOUtils; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.util.CommonClientUtils; import org.apache.avro.generic.IndexedRecord; @@ -54,7 +55,7 @@ public static HoodieCDCLogWriter createAvroCDCLogWri TaskContextSupplier taskContextSupplier, Supplier logWriterSupplier) { HoodieTableConfig tableConfig = hoodieTable.getMetaClient().getTableConfig(); - if (shouldWriteNativeCDCLogs(tableConfig)) { + if (CommonClientUtils.shouldWriteNativeLogs(config, tableConfig)) { return new HoodieAvroNativeCDCLogger( instantTime, config, @@ -78,8 +79,4 @@ public static HoodieCDCLogWriter createAvroCDCLogWri logWriterSupplier.get(), IOUtils.getMaxMemoryPerPartitionMerge(taskContextSupplier, config)); } - - public static boolean shouldWriteNativeCDCLogs(HoodieTableConfig tableConfig) { - return tableConfig.isLSMTreeStorageLayout(); - } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeCDCFileWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeCDCFileWriter.java index 9bb42cfdd19a6..11541273545ce 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeCDCFileWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeCDCFileWriter.java @@ -26,6 +26,8 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.cdc.HoodieCDCUtils; import org.apache.hudi.common.table.log.LogFileCreationCallback; +import org.apache.hudi.common.table.log.NativeLogFooterMetadata; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.core.io.storage.HoodieFileWriter; @@ -39,12 +41,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Properties; /** * Manages native CDC log file creation, rolling, writes, and stats. + * + * @param engine-native record representation, e.g. Avro {@code IndexedRecord} or Spark {@code InternalRow} */ -class HoodieNativeCDCFileWriter { +class HoodieNativeCDCFileWriter { private final String commitTime; private final String partitionPath; @@ -58,10 +61,10 @@ class HoodieNativeCDCFileWriter { private final LogFileCreationCallback fileCreationCallback; private final TaskContextSupplier taskContextSupplier; private final HoodieRecord.HoodieRecordType recordType; - private final Properties recordProperties; + private final Map cdcDataBlockMetadata; private final List cdcAbsPaths; private int nextLogVersion; - private HoodieFileWriter cdcWriter; + private HoodieFileWriter cdcWriter; HoodieNativeCDCFileWriter( String commitTime, @@ -88,15 +91,23 @@ class HoodieNativeCDCFileWriter { this.fileCreationCallback = fileCreationCallback; this.taskContextSupplier = taskContextSupplier; this.recordType = recordType; - this.recordProperties = new Properties(); - this.recordProperties.putAll(config.getProps()); + this.cdcDataBlockMetadata = new HashMap<>(); + this.cdcDataBlockMetadata.put(HeaderMetadataType.INSTANT_TIME, commitTime); + this.cdcDataBlockMetadata.put(HeaderMetadataType.SCHEMA, cdcSchema.toString()); this.cdcAbsPaths = new ArrayList<>(); this.nextLogVersion = HoodieLogFile.LOGFILE_BASE_VERSION; } - void write(String recordKey, HoodieRecord record) throws IOException { + /** + * Writes a CDC record in the engine-native representation. + * + *

The record is written as-is via {@link HoodieFileWriter#writeRow}. It must NOT be wrapped into a payload-based + * {@link HoodieRecord}: payloads like ExpressionPayload re-interpret the record bytes with their own schema + * (e.g. {@code hoodie.payload.record.schema}), which corrupts the CDC record that is already in its final form. + */ + void write(String recordKey, T record) throws IOException { ensureCDCWriter(); - cdcWriter.write(recordKey, record, cdcSchema, recordProperties); + cdcWriter.writeRow(recordKey, record); } Map getCDCWriteStats() { @@ -120,14 +131,16 @@ void close() throws IOException { } } + @SuppressWarnings("unchecked") private void ensureCDCWriter() throws IOException { if (cdcWriter != null && cdcWriter.canWrite()) { return; } close(); HoodieLogFile cdcLogFile = createNativeCDCLogFile(); - cdcWriter = HoodieFileWriterFactory.getFileWriter( + cdcWriter = (HoodieFileWriter) HoodieFileWriterFactory.getFileWriter( commitTime, cdcLogFile.getPath(), storage, config, cdcSchema, taskContextSupplier, recordType); + cdcWriter.addFooterMetadata(NativeLogFooterMetadata.toFooterMetadata(cdcDataBlockMetadata)); cdcAbsPaths.add(cdcLogFile.getPath()); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeCDCLogger.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeCDCLogger.java index 69f5009d3a61c..c13ba771555e0 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeCDCLogger.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeCDCLogger.java @@ -50,8 +50,6 @@ public class HoodieNativeCDCLogger implements HoodieCDCLogWriter implements HoodieCDCLogWriter recordContext; /** Manages native CDC file creation, rolling, writing, and stats. */ - private final HoodieNativeCDCFileWriter nativeCDCFileWriter; + private final HoodieNativeCDCFileWriter nativeCDCFileWriter; /** Last CDC record staged for write so it can be retracted if the merge later fails. */ private PendingCDCRecord pendingRecord; @@ -82,13 +80,12 @@ public HoodieNativeCDCLogger( RecordContext recordContext, HoodieRecord.HoodieRecordType recordType) { this.commitTime = commitTime; - this.partitionPath = partitionPath; this.dataSchema = HoodieSchemaCache.intern(HoodieSchemaUtils.removeMetadataFields(schema)); this.cdcSupplementalLoggingMode = tableConfig.cdcSupplementalLoggingMode(); this.cdcSchema = HoodieCDCUtils.schemaBySupplementalLoggingMode(cdcSupplementalLoggingMode, dataSchema); this.recordContext = recordContext; this.cdcSchemaId = recordContext.encodeSchema(cdcSchema); - this.nativeCDCFileWriter = new HoodieNativeCDCFileWriter( + this.nativeCDCFileWriter = new HoodieNativeCDCFileWriter<>( commitTime, partitionPath, storage, @@ -186,9 +183,9 @@ private void flushPendingRecord() { return; } try { - nativeCDCFileWriter.write( - pendingRecord.recordKey, - recordContext.constructHoodieRecord(pendingRecord.record, partitionPath)); + // Write the engine-native CDC record directly; wrapping it into a payload-based HoodieRecord + // would let the payload (e.g., ExpressionPayload) re-interpret the CDC record with a wrong schema. + nativeCDCFileWriter.write(pendingRecord.recordKey, pendingRecord.record.getRecord()); pendingRecord = null; } catch (IOException e) { throw new HoodieException("Failed to write the cdc data to native cdc log file", e); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeLogFormatWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeLogFormatWriter.java index e576a6485b787..09c3608a53654 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeLogFormatWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/cdc/HoodieNativeLogFormatWriter.java @@ -24,17 +24,17 @@ import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemas; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.AppendResult; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.LogFileCreationCallback; +import org.apache.hudi.common.table.log.LogReaderUtils; import org.apache.hudi.common.table.log.NativeLogFooterMetadata; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import org.apache.hudi.common.table.read.BufferedRecord; -import org.apache.hudi.common.table.read.BufferedRecords; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.OrderingValues; import org.apache.hudi.common.util.collection.ArrayComparable; @@ -47,9 +47,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import static org.apache.hudi.common.model.LogExtensions.DATA_LOG_EXTENSION; import static org.apache.hudi.common.model.LogExtensions.DELETE_LOG_EXTENSION; @@ -66,12 +69,15 @@ public class HoodieNativeLogFormatWriter extends HoodieLogFormat.Writer { private final RecordContext recordContext; private final List orderingFieldNames; private final Properties recordProperties; + private final Option baseFileInstantTimeOfPositions; private HoodieFileWriter dataFileWriter; private HoodieFileWriter deleteFileWriter; private HoodieLogFile dataLogFile; private HoodieLogFile deleteLogFile; private HoodieSchema deleteLogSchema; private int currentAppendVersion = -1; + private final List dataRecordPositions = new ArrayList<>(); + private final List deleteRecordPositions = new ArrayList<>(); private List lastAppendResults = new ArrayList<>(); private Option lastDataFileFormatMetadata = Option.empty(); @@ -90,7 +96,8 @@ public HoodieNativeLogFormatWriter(Integer bufferSize, HoodieSchema tableSchema, TaskContextSupplier taskContextSupplier, RecordContext recordContext, - List orderingFieldNames) throws IOException { + List orderingFieldNames, + Option baseFileInstantTimeOfPositions) throws IOException { super(bufferSize, storage, parentPath, logFileId, DATA_LOG_EXTENSION, instantTime, logVersion, logWriteToken, null, 0L, sizeThreshold, fileCreationCallback, tableVersion); this.writeConfig = writeConfig; @@ -102,6 +109,7 @@ public HoodieNativeLogFormatWriter(Integer bufferSize, this.recordProperties = new Properties(); this.recordProperties.putAll(writeConfig.getProps()); this.logFile = new HoodieLogFile(makeNativeLogPath(this.logVersion, DATA_LOG_EXTENSION), this.fileSize); + this.baseFileInstantTimeOfPositions = baseFileInstantTimeOfPositions; } public List getLastAppendResults() { @@ -157,22 +165,25 @@ public void appendRecord(HoodieRecord record, HoodieSchema recordSchema, String ensureDataFileWriter(recordSchema); dataFileWriter.write(record.getRecordKey(recordSchema, keyFieldName), record, recordSchema, recordProperties); + dataRecordPositions.add(record.getCurrentPosition()); } public void appendDeleteRecord(HoodieRecord record, HoodieSchema recordSchema, String keyFieldName) throws IOException { ensureDeleteFileWriter(); String recordKey = record.getRecordKey(recordSchema, keyFieldName); - Comparable orderingValue = record.getOrderingValue( - recordSchema, recordProperties, orderingFieldNames.toArray(new String[0])); + Comparable orderingValue = getDeleteOrderingValue(record, recordSchema); Object deleteEngineRecord = recordContext.constructEngineRecord( deleteLogSchema, createDeleteLogFieldValues(recordKey, orderingValue)); - // Keep isDelete=false here so RecordContext constructs a data-bearing HoodieRecord - // with the native delete-log row. The delete semantics come from the delete log file - // itself; setting isDelete=true would create a HoodieEmptyRecord and lose the row. - BufferedRecord deleteRecord = BufferedRecords.fromEngineRecord( - deleteEngineRecord, deleteLogSchema, recordContext, orderingValue, recordKey, false); - deleteFileWriter.write(recordKey, recordContext.constructHoodieRecord(deleteRecord, record.getPartitionPath()), - deleteLogSchema, recordProperties); + deleteFileWriter.writeRow(recordKey, deleteEngineRecord); + long recordPosition = baseFileInstantTimeOfPositions.isPresent() ? record.getCurrentPosition() : -1L; + deleteRecordPositions.add(recordPosition); + } + + private Comparable getDeleteOrderingValue(HoodieRecord record, HoodieSchema recordSchema) { + if (orderingFieldNames.isEmpty()) { + return OrderingValues.getDefault(); + } + return record.getOrderingValue(recordSchema, recordProperties, orderingFieldNames.toArray(new String[0])); } private Object[] createDeleteLogFieldValues(String recordKey, Comparable orderingValue) { @@ -211,8 +222,40 @@ public AppendResult flushAppend(Map header) throws I private void addFooterMetadata(Map header) throws IOException { if (dataFileWriter != null) { - dataFileWriter.addFooterMetadata(NativeLogFooterMetadata.toFooterMetadata(header)); + dataFileWriter.addFooterMetadata(NativeLogFooterMetadata.toFooterMetadata( + addRecordPositionsIfRequired(header, dataRecordPositions))); + } + if (deleteFileWriter != null) { + deleteFileWriter.addFooterMetadata(NativeLogFooterMetadata.toFooterMetadata( + addRecordPositionsIfRequired(header, deleteRecordPositions))); + } + } + + private Map addRecordPositionsIfRequired( + Map header, List recordPositions) throws IOException { + if (!header.containsKey(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS)) { + return header; + } + + Set positionSet = new HashSet<>(recordPositions.size()); + long previousPosition = Long.MIN_VALUE; + for (Long position : recordPositions) { + // RECORD_POSITIONS is encoded as a bitmap and decoded in ascending order. Native log records + // are read back in file write order, so only publish positions when both orders match. + if (!HoodieRecordLocation.isPositionValid(position) || position <= previousPosition) { + Map updatedHeader = new HashMap<>(header); + updatedHeader.remove(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS); + return updatedHeader; + } + previousPosition = position; + positionSet.add(position); } + + Map updatedHeader = new HashMap<>(header); + if (positionSet.size() == recordPositions.size()) { + updatedHeader.put(HeaderMetadataType.RECORD_POSITIONS, LogReaderUtils.encodePositions(positionSet)); + } + return updatedHeader; } private void ensureDataFileWriter(HoodieSchema recordSchema) throws IOException { @@ -230,9 +273,13 @@ private void ensureDeleteFileWriter() throws IOException { if (deleteFileWriter == null) { deleteLogFile = createNativeLogFile(currentAppendVersion, DELETE_LOG_EXTENSION); deleteLogSchema = HoodieSchemas.createDeleteLogSchema(tableSchema, orderingFieldNames); + // The delete records are built through recordContext#constructEngineRecord (see #appendDeleteRecord), so the + // writer must match the record context's engine type rather than the merger's record type: on Spark executors + // the reader context for write degrades to Avro (no engine context available), and using the merger record + // type here (e.g. SPARK) would create an engine-native writer that fails on the Avro delete records. deleteFileWriter = HoodieFileWriterFactory.getFileWriter( instantTime, deleteLogFile.getPath(), storage, writeConfig, deleteLogSchema, taskContextSupplier, - writeConfig.getRecordMerger().getRecordType()); + recordContext.getEngineRecordType()); } } @@ -245,13 +292,16 @@ private void ensureAppendVersion() throws IOException { private void closeFileWriters() throws IOException { if (dataFileWriter != null) { dataFileWriter.close(); - lastDataFileFormatMetadata = Option.ofNullable(dataFileWriter.getFileFormatMetadata()); + lastDataFileFormatMetadata = writeConfig.isMetadataColumnStatsIndexEnabled() + ? Option.ofNullable(dataFileWriter.getFileFormatMetadata()) : Option.empty(); dataFileWriter = null; } if (deleteFileWriter != null) { deleteFileWriter.close(); deleteFileWriter = null; } + dataRecordPositions.clear(); + deleteRecordPositions.clear(); } private HoodieLogFile createNativeLogFile(int version, String logExtension) throws IOException { @@ -272,10 +322,11 @@ private AppendResult completeAppend(int appendVersion) { return new AppendResult(logFile, 0, 0); } - long totalSize = lastAppendResults.stream().mapToLong(AppendResult::size).sum(); + // The returned result preserves the legacy single-result writer contract. Native append handles consume + // lastAppendResults directly so every physical data/delete file gets its own write stat. AppendResult firstResult = lastAppendResults.get(0); this.logFile = firstResult.logFile(); - return new AppendResult(firstResult.logFile(), 0, totalSize); + return firstResult; } private int nextAvailableVersion() throws IOException { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java index 940f0e831e090..07a9c2bda9960 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java @@ -47,6 +47,7 @@ import org.apache.hudi.io.HoodieMergeHandle; import org.apache.hudi.io.HoodieMergeHandleFactory; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.util.CommonClientUtils; import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.IndexedRecord; @@ -173,7 +174,8 @@ public List logCompact(HoodieWriteConfig writeConfig, TaskContextSupplier taskContextSupplier) throws IOException { HoodieReaderContext readerContext = new HoodieAvroReaderContext( table.getStorageConf(), table.getMetaClient().getTableConfig(), instantRange, Option.empty(), writeConfig.getProps()); - HoodieAppendHandle appendHandle = table.getMetaClient().getTableConfig().isLSMTreeStorageLayout() + HoodieAppendHandle appendHandle = CommonClientUtils.shouldWriteNativeLogs( + writeConfig, table.getMetaClient().getTableConfig()) ? new FileGroupReaderBasedNativeLogAppendHandle<>(writeConfig, instantTime, table, operation, taskContextSupplier, readerContext) : new FileGroupReaderBasedInlineLogAppendHandle<>(writeConfig, instantTime, table, operation, taskContextSupplier, readerContext); appendHandle.doAppend(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToTenUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToTenUpgradeHandler.java new file mode 100644 index 0000000000000..97fdea90036fa --- /dev/null +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToTenUpgradeHandler.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.upgrade; + +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.config.HoodieWriteConfig; + +/** + * Version 10 enables native log format by default for new writes. Existing version 9 + * inline log files remain readable by version 10 readers, so there is no table metadata + * rewrite required for the upgrade. + */ +public class NineToTenUpgradeHandler implements UpgradeHandler { + + @Override + public UpgradeDowngrade.TableConfigChangeSet upgrade( + HoodieWriteConfig config, + HoodieEngineContext context, + String instantTime, + SupportsUpgradeDowngrade upgradeDowngradeHelper) { + return new UpgradeDowngrade.TableConfigChangeSet(); + } +} diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java index febb815ad34fd..a550fbe706b1d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java @@ -302,6 +302,8 @@ protected UpgradeDowngrade.TableConfigChangeSet upgrade(HoodieTableVersion fromV return new SevenToEightUpgradeHandler().upgrade(config, context, instantTime, upgradeDowngradeHelper); } else if (fromVersion == HoodieTableVersion.EIGHT && toVersion == HoodieTableVersion.NINE) { return new EightToNineUpgradeHandler().upgrade(config, context, instantTime, upgradeDowngradeHelper); + } else if (fromVersion == HoodieTableVersion.NINE && toVersion == HoodieTableVersion.TEN) { + return new NineToTenUpgradeHandler().upgrade(config, context, instantTime, upgradeDowngradeHelper); } else { throw new HoodieUpgradeDowngradeException(fromVersion.versionCode(), toVersion.versionCode(), true); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/CommonClientUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/CommonClientUtils.java index cd95b7ba5e133..f71e0b9bead74 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/CommonClientUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/CommonClientUtils.java @@ -21,6 +21,7 @@ package org.apache.hudi.util; import org.apache.hudi.avro.model.HoodieClusteringPlan; +import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieCommitMetadata; @@ -64,6 +65,16 @@ public static void validateTableVersion(HoodieTableConfig tableConfig, HoodieWri if (tableConfig.getTableVersion().lesserThan(HoodieTableVersion.EIGHT) && writeConfig.isNonBlockingConcurrencyControl()) { throw new HoodieNotSupportedException("Non-blocking concurrency control is not supported for table versions < 8."); } + + // The LSM-tree storage layout depends on the native (v2) log format, which is only written for + // writer versions >= TEN (see #shouldWriteNativeLogFormat). Reject an LSM-tree table whose writer + // version is below TEN, otherwise writes would silently fall back to the inline log format. + if (tableConfig.isLSMTreeStorageLayout() + && writeConfig.getWriteVersion().lesserThan(HoodieTableVersion.TEN)) { + throw new HoodieNotSupportedException(String.format( + "LSM-tree storage layout requires writer version >= %s (found writer version %s).", + HoodieTableVersion.TEN, writeConfig.getWriteVersion())); + } } public static boolean areTableVersionsCompatible(HoodieTableVersion tableVersion, HoodieTableVersion writeVersion) { @@ -120,6 +131,41 @@ public static HoodieLogBlock.HoodieLogBlockType getLogBlockType(HoodieWriteConfi } } + /** + * Whether log blocks should be written in the native (v2) log format (standalone native + * files written via {@code HoodieNativeLogFormatWriter}) instead of the legacy inline + * log format. The native format is the default for write version >= {@link HoodieTableVersion#TEN}, + * except for Lance base files. + * + *

This decision is keyed on the effective write version (i.e. {@code HoodieWriteConfig#getWriteVersion()}) + * and the effective base file format, consistent with how the inline log block layout is + * selected, so that the on-disk format follows what the writer is targeting during + * upgrade/downgrade windows. Lance remains on the legacy inline log format until native Lance log + * support is complete. + * + * @param writeConfig the writer configuration. + * @param tableConfig the persisted table configuration. + */ + public static boolean shouldWriteNativeLogs(HoodieWriteConfig writeConfig, HoodieTableConfig tableConfig) { + if (getBaseFileFormat(writeConfig, tableConfig) == HoodieFileFormat.LANCE) { + return false; + } + return writeConfig.getWriteVersion().greaterThanOrEquals(HoodieTableVersion.TEN); + } + + public static void validateIndexSupportForNativeLogFormat(HoodieWriteConfig writeConfig, HoodieFileFormat nativeLogFileFormat) { + if (!writeConfig.isMetadataColumnStatsIndexEnabled()) { + return; + } + + if (nativeLogFileFormat == HoodieFileFormat.ORC) { + throw new HoodieNotSupportedException(String.format( + "Column stats index is not supported for native %s log files because its HoodieFileWriter does not support file format metadata. " + + "Please disable %s.", + nativeLogFileFormat, HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key())); + } + } + public static String generateWriteToken(TaskContextSupplier taskContextSupplier) { try { return FSUtils.makeWriteToken( diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java index 2c43119c5eeeb..8d50ec397ccc5 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java @@ -98,7 +98,7 @@ public void testPropertyLoading(boolean withAlternative) throws IOException { @Test public void testSupportedTableWriteVersions() { Set supportedVersions = CollectionUtils.createSet( - HoodieTableVersion.SIX, HoodieTableVersion.EIGHT, HoodieTableVersion.NINE + HoodieTableVersion.SIX, HoodieTableVersion.EIGHT, HoodieTableVersion.NINE, HoodieTableVersion.TEN ); Arrays.stream(HoodieTableVersion.values()) .filter(version -> !supportedVersions.contains(version)) diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/TestSecondaryIndexStreamingTracker.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/TestSecondaryIndexStreamingTracker.java index 0ca5108cd4bb5..e129b46642eeb 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/TestSecondaryIndexStreamingTracker.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/TestSecondaryIndexStreamingTracker.java @@ -438,4 +438,4 @@ private void verifyUpdateEntries(WriteStatus status, String recordKey, assertTrue(foundDelete && foundInsert, "Should have both delete and insert entries"); }); } -} \ No newline at end of file +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/cdc/TestHoodieNativeCDCFileWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/cdc/TestHoodieNativeCDCFileWriter.java new file mode 100644 index 0000000000000..d7d3ae876917a --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/cdc/TestHoodieNativeCDCFileWriter.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.cdc; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.TaskContextSupplier; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.log.LogFileCreationCallback; +import org.apache.hudi.common.table.log.NativeLogFooterMetadata; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.core.io.storage.HoodieFileWriter; +import org.apache.hudi.core.io.storage.HoodieFileWriterFactory; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; + +import org.apache.avro.generic.IndexedRecord; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestHoodieNativeCDCFileWriter { + + @Test + public void testAddsLogBlockHeaderToFooterMetadata() throws Exception { + String instantTime = "100"; + String schemaString = "{\"type\":\"record\",\"name\":\"cdc\",\"fields\":[]}"; + HoodieStorage storage = mock(HoodieStorage.class); + HoodieWriteConfig config = mock(HoodieWriteConfig.class); + HoodieSchema cdcSchema = mock(HoodieSchema.class); + HoodieFileWriter fileWriter = mock(HoodieFileWriter.class); + StoragePath parentPath = new StoragePath("/tmp/partition"); + + when(config.getProps()).thenReturn(new TypedProperties()); + when(cdcSchema.toString()).thenReturn(schemaString); + when(storage.exists(any(StoragePath.class))).thenReturn(false); + + try (MockedStatic writerFactory = mockStatic(HoodieFileWriterFactory.class)) { + writerFactory.when(() -> HoodieFileWriterFactory.getFileWriter( + eq(instantTime), any(StoragePath.class), eq(storage), eq(config), eq(cdcSchema), + any(TaskContextSupplier.class), eq(HoodieRecord.HoodieRecordType.AVRO))) + .thenReturn(fileWriter); + + HoodieNativeCDCFileWriter writer = new HoodieNativeCDCFileWriter<>( + instantTime, + "partition", + storage, + config, + cdcSchema, + HoodieFileFormat.PARQUET, + parentPath, + "file-1", + "1-0-1", + new LogFileCreationCallback() { + }, + mock(TaskContextSupplier.class), + HoodieRecord.HoodieRecordType.AVRO); + + writer.write("key1", mock(IndexedRecord.class)); + } + + ArgumentCaptor> footerCaptor = ArgumentCaptor.forClass(Map.class); + verify(fileWriter).addFooterMetadata(footerCaptor.capture()); + Map header = + NativeLogFooterMetadata.fromFooterMetadata(footerCaptor.getValue()); + assertEquals(instantTime, header.get(HeaderMetadataType.INSTANT_TIME)); + assertEquals(schemaString, header.get(HeaderMetadataType.SCHEMA)); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/cdc/TestHoodieNativeLogFormatWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/cdc/TestHoodieNativeLogFormatWriter.java new file mode 100644 index 0000000000000..7c38d7520d20b --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/cdc/TestHoodieNativeLogFormatWriter.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.cdc; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.RecordContext; +import org.apache.hudi.common.engine.TaskContextSupplier; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.log.LogFileCreationCallback; +import org.apache.hudi.common.table.log.LogReaderUtils; +import org.apache.hudi.common.table.log.NativeLogFooterMetadata; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.OrderingValues; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.core.io.storage.HoodieFileWriter; +import org.apache.hudi.core.io.storage.HoodieFileWriterFactory; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; + +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; +import org.roaringbitmap.longlong.Roaring64NavigableMap; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestHoodieNativeLogFormatWriter { + + @Test + public void testAddsRecordPositionsToDataLogFooter() throws Exception { + Map parsedHeader = writeDataLogFooterWithPositions(2L, 7L); + + assertEquals("001", parsedHeader.get(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS)); + assertEquals(Arrays.asList(2L, 7L), + toList(LogReaderUtils.decodeRecordPositionsHeader(parsedHeader.get(HeaderMetadataType.RECORD_POSITIONS)))); + } + + @Test + public void testSkipsOutOfOrderRecordPositions() throws Exception { + Map parsedHeader = writeDataLogFooterWithPositions(7L, 2L); + + assertFalse(parsedHeader.containsKey(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS)); + assertFalse(parsedHeader.containsKey(HeaderMetadataType.RECORD_POSITIONS)); + } + + @Test + public void testRemovesBaseFileInstantForInvalidRecordPosition() throws Exception { + Map parsedHeader = writeDataLogFooterWithPositions(7L, -1L); + + assertFalse(parsedHeader.containsKey(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS)); + assertFalse(parsedHeader.containsKey(HeaderMetadataType.RECORD_POSITIONS)); + } + + @Test + public void testSkipsDuplicateRecordPositions() throws Exception { + Map parsedHeader = writeDataLogFooterWithPositions(7L, 7L); + + assertNull(parsedHeader.get(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS)); + assertFalse(parsedHeader.containsKey(HeaderMetadataType.RECORD_POSITIONS)); + } + + @Test + public void testAddsRecordPositionsToDeleteLogFooter() throws Exception { + Map parsedHeader = writeDeleteLogFooterWithPositions(Option.of("001")); + + assertEquals("001", parsedHeader.get(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS)); + assertEquals(Arrays.asList(7L), + toList(LogReaderUtils.decodeRecordPositionsHeader(parsedHeader.get(HeaderMetadataType.RECORD_POSITIONS)))); + } + + @Test + public void testSkipsDeleteRecordPositionsWithoutBaseFileInstant() throws Exception { + Map parsedHeader = writeDeleteLogFooterWithPositions(Option.empty()); + + assertFalse(parsedHeader.containsKey(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS)); + assertFalse(parsedHeader.containsKey(HeaderMetadataType.RECORD_POSITIONS)); + } + + @Test + public void testUsesDefaultOrderingValueForCommitTimeDeleteLog() throws Exception { + HoodieSchema schema = mock(HoodieSchema.class); + HoodieRecord record = mock(HoodieRecord.class); + when(record.getRecordKey(schema, HoodieRecord.RECORD_KEY_METADATA_FIELD)).thenReturn("key-1"); + when(record.getCurrentPosition()).thenReturn(7L); + doReturn(HoodieRecord.DEFAULT_ORDERING_VALUE + 1).when(record).getOrderingValue(eq(schema), any(), any()); + + writeDeleteLogFooterWithPositions(Option.empty(), record, schema, new ArrayList<>()); + + verify(record, never()).getOrderingValue(eq(schema), any(), any()); + } + + private static Map writeDataLogFooterWithPositions(long... positions) throws Exception { + String instantTime = "100"; + String schemaString = "{\"type\":\"record\",\"name\":\"test\",\"fields\":[]}"; + HoodieStorage storage = mock(HoodieStorage.class); + HoodieWriteConfig config = mock(HoodieWriteConfig.class); + HoodieSchema schema = mock(HoodieSchema.class); + HoodieRecordMerger merger = mock(HoodieRecordMerger.class); + HoodieFileWriter fileWriter = mock(HoodieFileWriter.class); + StoragePath parentPath = new StoragePath("/tmp/partition"); + + when(config.getProps()).thenReturn(new TypedProperties()); + when(config.getRecordMerger()).thenReturn(merger); + when(merger.getRecordType()).thenReturn(HoodieRecord.HoodieRecordType.AVRO); + when(storage.exists(any(StoragePath.class))).thenReturn(false); + when(storage.getPathInfo(any(StoragePath.class))).thenAnswer(invocation -> + new StoragePathInfo(invocation.getArgument(0), 1L, false, (short) 1, 1L, 1L)); + + try (MockedStatic writerFactory = mockStatic(HoodieFileWriterFactory.class)) { + writerFactory.when(() -> HoodieFileWriterFactory.getFileWriter( + eq(instantTime), any(StoragePath.class), eq(storage), eq(config), eq(schema), + any(TaskContextSupplier.class), eq(HoodieRecord.HoodieRecordType.AVRO))) + .thenReturn(fileWriter); + + HoodieNativeLogFormatWriter writer = new HoodieNativeLogFormatWriter( + 4096, + storage, + parentPath, + "file-1", + instantTime, + 1, + "1-0-1", + 1024L, + new LogFileCreationCallback() { + }, + HoodieTableVersion.current(), + config, + HoodieFileFormat.PARQUET, + schema, + mock(TaskContextSupplier.class), + mock(RecordContext.class), + new ArrayList<>(), + Option.of("001")); + + for (int i = 0; i < positions.length; i++) { + writer.appendRecord(recordWithPosition("key-" + i, positions[i], schema), + schema, HoodieRecord.RECORD_KEY_METADATA_FIELD); + } + + Map header = new HashMap<>(); + header.put(HeaderMetadataType.SCHEMA, schemaString); + header.put(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS, "001"); + writer.flushAppend(header); + } + + ArgumentCaptor> footerCaptor = ArgumentCaptor.forClass(Map.class); + verify(fileWriter).addFooterMetadata(footerCaptor.capture()); + return NativeLogFooterMetadata.fromFooterMetadata(footerCaptor.getValue()); + } + + private static Map writeDeleteLogFooterWithPositions( + Option baseFileInstantTimeOfPositions) throws Exception { + HoodieSchema schema = mock(HoodieSchema.class); + return writeDeleteLogFooterWithPositions( + baseFileInstantTimeOfPositions, recordWithPosition("key-1", 7L, schema), schema, new ArrayList<>()); + } + + private static Map writeDeleteLogFooterWithPositions( + Option baseFileInstantTimeOfPositions, HoodieRecord record, HoodieSchema schema, + List orderingFieldNames) throws Exception { + String instantTime = "100"; + String schemaString = "{\"type\":\"record\",\"name\":\"test\",\"fields\":[]}"; + HoodieStorage storage = mock(HoodieStorage.class); + HoodieWriteConfig config = mock(HoodieWriteConfig.class); + HoodieRecordMerger merger = mock(HoodieRecordMerger.class); + HoodieFileWriter fileWriter = mock(HoodieFileWriter.class); + RecordContext recordContext = mock(RecordContext.class); + StoragePath parentPath = new StoragePath("/tmp/partition"); + + when(config.getProps()).thenReturn(new TypedProperties()); + when(config.getRecordMerger()).thenReturn(merger); + when(merger.getRecordType()).thenReturn(HoodieRecord.HoodieRecordType.AVRO); + when(recordContext.getEngineRecordType()).thenReturn(HoodieRecord.HoodieRecordType.AVRO); + when(recordContext.convertValueToEngineType(any())).thenAnswer(invocation -> invocation.getArgument(0)); + when(recordContext.constructEngineRecord(any(HoodieSchema.class), any(Object[].class))) + .thenReturn(new Object()); + when(storage.exists(any(StoragePath.class))).thenReturn(false); + when(storage.getPathInfo(any(StoragePath.class))).thenAnswer(invocation -> + new StoragePathInfo(invocation.getArgument(0), 1L, false, (short) 1, 1L, 1L)); + + try (MockedStatic writerFactory = mockStatic(HoodieFileWriterFactory.class)) { + writerFactory.when(() -> HoodieFileWriterFactory.getFileWriter( + eq(instantTime), any(StoragePath.class), eq(storage), eq(config), any(HoodieSchema.class), + any(TaskContextSupplier.class), eq(HoodieRecord.HoodieRecordType.AVRO))) + .thenReturn(fileWriter); + + HoodieNativeLogFormatWriter writer = new HoodieNativeLogFormatWriter( + 4096, + storage, + parentPath, + "file-1", + instantTime, + 1, + "1-0-1", + 1024L, + new LogFileCreationCallback() { + }, + HoodieTableVersion.current(), + config, + HoodieFileFormat.PARQUET, + schema, + mock(TaskContextSupplier.class), + recordContext, + orderingFieldNames, + baseFileInstantTimeOfPositions); + + writer.appendDeleteRecord(record, schema, HoodieRecord.RECORD_KEY_METADATA_FIELD); + + Map header = new HashMap<>(); + header.put(HeaderMetadataType.SCHEMA, schemaString); + baseFileInstantTimeOfPositions.ifPresent(baseInstantTime -> + header.put(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS, baseInstantTime)); + writer.flushAppend(header); + } + + ArgumentCaptor> footerCaptor = ArgumentCaptor.forClass(Map.class); + verify(fileWriter).addFooterMetadata(footerCaptor.capture()); + return NativeLogFooterMetadata.fromFooterMetadata(footerCaptor.getValue()); + } + + private static HoodieRecord recordWithPosition(String key, long position, HoodieSchema schema) throws Exception { + HoodieRecord record = mock(HoodieRecord.class); + when(record.getRecordKey(schema, HoodieRecord.RECORD_KEY_METADATA_FIELD)).thenReturn(key); + when(record.getCurrentPosition()).thenReturn(position); + when(record.getOrderingValue(eq(schema), any(), any())).thenReturn(OrderingValues.getDefault()); + return record; + } + + private static List toList(Roaring64NavigableMap positions) { + List values = new ArrayList<>(); + positions.iterator().forEachRemaining(values::add); + return values; + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestFileWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestFileWriter.java index 154859bc78c42..871189c224b67 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestFileWriter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestFileWriter.java @@ -28,7 +28,7 @@ import java.io.IOException; import java.util.Properties; -public class TestFileWriter implements HoodieFileWriter { +public class TestFileWriter implements HoodieFileWriter { private boolean closed = false; private boolean failOnWrite = false; @@ -69,6 +69,16 @@ public void write(String recordKey, HoodieRecord record, HoodieSchema schema, Pr } } + @Override + public void writeRow(String recordKey, Object record) throws IOException { + if (closed) { + throw new IOException("Writer is closed"); + } + if (failOnWrite) { + throw new IOException("Simulated file writer write failure"); + } + } + @Override public void close() throws IOException { closed = true; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java index 5bc01c6519f33..c7f9255ec51b8 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java @@ -21,6 +21,7 @@ package org.apache.hudi.utils; import org.apache.hudi.common.engine.TaskContextSupplier; +import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.config.HoodieWriteConfig; @@ -36,6 +37,7 @@ import static org.apache.hudi.util.CommonClientUtils.areTableVersionsCompatible; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -68,6 +70,30 @@ void testDisallowNBCCPreVersion8() { assertThrows(HoodieNotSupportedException.class, () -> CommonClientUtils.validateTableVersion(tConfig, wConfig)); } + @ParameterizedTest(name = "LSM-tree layout, writer version {0} should throw: {1}") + @MethodSource("provideLSMTreeVersionExpectations") + void testLSMTreeStorageLayoutRequiresVersionTen(HoodieTableVersion writeVersion, boolean expectThrow) { + HoodieWriteConfig wConfig = mock(HoodieWriteConfig.class); + HoodieTableConfig tConfig = mock(HoodieTableConfig.class); + when(wConfig.getWriteVersion()).thenReturn(writeVersion); + when(tConfig.getTableVersion()).thenReturn(writeVersion); + when(tConfig.isLSMTreeStorageLayout()).thenReturn(true); + + if (expectThrow) { + assertThrows(HoodieNotSupportedException.class, () -> CommonClientUtils.validateTableVersion(tConfig, wConfig)); + } else { + CommonClientUtils.validateTableVersion(tConfig, wConfig); + } + } + + private static Stream provideLSMTreeVersionExpectations() { + return Stream.of( + Arguments.of(HoodieTableVersion.EIGHT, true), + Arguments.of(HoodieTableVersion.NINE, true), + Arguments.of(HoodieTableVersion.TEN, false) + ); + } + @Test void testGenerateTokenOnError() { // given: a task context supplies that throws errors. @@ -78,6 +104,64 @@ void testGenerateTokenOnError() { assertEquals("0-0-0", CommonClientUtils.generateWriteToken(taskContextSupplier)); } + @ParameterizedTest(name = "Write version {0} with base file format {1} should write native log format: {2}") + @MethodSource("provideWriteVersionNativeLogExpectations") + void testShouldWriteNativeLogs(HoodieTableVersion writeVersion, HoodieFileFormat baseFileFormat, boolean expected) { + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(writeConfig.getWriteVersion()).thenReturn(writeVersion); + when(tableConfig.getBaseFileFormat()).thenReturn(baseFileFormat); + + assertEquals(expected, CommonClientUtils.shouldWriteNativeLogs(writeConfig, tableConfig)); + } + + private static Stream provideWriteVersionNativeLogExpectations() { + // Native log format is the default for write version >= TEN, except for Lance base files. + return Stream.of( + Arguments.of(HoodieTableVersion.SIX, HoodieFileFormat.PARQUET, false), + Arguments.of(HoodieTableVersion.EIGHT, HoodieFileFormat.PARQUET, false), + Arguments.of(HoodieTableVersion.NINE, HoodieFileFormat.PARQUET, false), + Arguments.of(HoodieTableVersion.TEN, HoodieFileFormat.PARQUET, true), + Arguments.of(HoodieTableVersion.TEN, HoodieFileFormat.ORC, true), + Arguments.of(HoodieTableVersion.TEN, HoodieFileFormat.LANCE, false) + ); + } + + @Test + void testShouldWriteInlineLogFormatForMultiFormatLanceWrites() { + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(writeConfig.getWriteVersion()).thenReturn(HoodieTableVersion.TEN); + when(writeConfig.contains(HoodieWriteConfig.BASE_FILE_FORMAT)).thenReturn(true); + when(writeConfig.getBaseFileFormat()).thenReturn(HoodieFileFormat.LANCE); + when(tableConfig.isMultipleBaseFileFormatsEnabled()).thenReturn(true); + when(tableConfig.getBaseFileFormat()).thenReturn(HoodieFileFormat.PARQUET); + + assertFalse(CommonClientUtils.shouldWriteNativeLogs(writeConfig, tableConfig)); + } + + @ParameterizedTest(name = "{0} native log with column stats enabled {1} should throw: {2}") + @MethodSource("provideNativeLogColumnStatsExpectations") + void testValidateIndexSupportForNativeLogFormat( + HoodieFileFormat nativeLogFileFormat, boolean enableColumnStats, boolean expectThrow) { + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + when(writeConfig.isMetadataColumnStatsIndexEnabled()).thenReturn(enableColumnStats); + + if (expectThrow) { + assertThrows(HoodieNotSupportedException.class, + () -> CommonClientUtils.validateIndexSupportForNativeLogFormat(writeConfig, nativeLogFileFormat)); + } else { + CommonClientUtils.validateIndexSupportForNativeLogFormat(writeConfig, nativeLogFileFormat); + } + } + + private static Stream provideNativeLogColumnStatsExpectations() { + return Stream.of( + Arguments.of(HoodieFileFormat.ORC, true, true), + Arguments.of(HoodieFileFormat.ORC, false, false) + ); + } + @ParameterizedTest(name = "Table version {0} with write version {1} should be valid: {2}") @MethodSource("provideValidTableVersionWriteVersionPairs") void testValidTableVersionWriteVersionPairs( diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java index 40ad0c8c50005..483a8e8687eac 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.io.v2.RowDataInlineLogWriteHandle; import org.apache.hudi.io.v2.RowDataNativeLogWriteHandle; @@ -30,6 +31,7 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.commit.BucketInfo; import org.apache.hudi.table.action.commit.BucketType; +import org.apache.hudi.util.CommonClientUtils; import org.apache.hadoop.fs.Path; @@ -326,7 +328,10 @@ public static DeltaCommitWriteHandleFactory getInstance final String fileID = bucketInfo.getFileIdPrefix(); final String partitionPath = bucketInfo.getPartitionPath(); final TaskContextSupplier contextSupplier = table.getTaskContextSupplier(); - if (table.getMetaClient().getTableConfig().isLSMTreeStorageLayout()) { + if (CommonClientUtils.shouldWriteNativeLogs(config, table.getMetaClient().getTableConfig())) { + if (table.requireSortedRecords()) { + recordItr = HoodieRecordUtils.sortRecordsByRecordKey(recordItr); + } return new FlinkNativeLogAppendHandle<>(config, instantTime, table, partitionPath, fileID, bucketInfo.getBucketType(), recordItr, contextSupplier); } @@ -354,7 +359,7 @@ public static DeltaCommitRowDataHandleFactory getInstan String instantTime, HoodieTable table, Iterator> recordIterator) { - if (table.getMetaClient().getTableConfig().isLSMTreeStorageLayout()) { + if (CommonClientUtils.shouldWriteNativeLogs(config, table.getMetaClient().getTableConfig())) { return new RowDataNativeLogWriteHandle<>( config, instantTime, diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriter.java index a9aa32decd6ad..f14e97e511bc5 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriter.java @@ -31,20 +31,13 @@ /** * Abstraction to assist in writing {@link RowData}s to be used in datasource implementation. */ -public interface HoodieRowDataFileWriter extends HoodieFileWriter { +public interface HoodieRowDataFileWriter extends HoodieFileWriter { /** * Returns {@code true} if this RowFileWriter can take in more writes. else {@code false}. */ boolean canWrite(); - /** - * Writes an {@link RowData} to the {@link HoodieRowDataFileWriter}. Also takes in associated record key to be added to bloom filter if required. - * - * @throws IOException on any exception while writing. - */ - void writeRow(String key, RowData row) throws IOException; - /** * Writes an {@link RowData} into the {@link HoodieRowDataFileWriter} with metadata fields. * Also takes in associated record key to be added to bloom filter if required. diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkMergeOnReadTable.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkMergeOnReadTable.java index d6d9b749f7a77..df73654db7411 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkMergeOnReadTable.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkMergeOnReadTable.java @@ -44,6 +44,7 @@ import org.apache.hudi.table.action.compact.ScheduleCompactionActionExecutor; import org.apache.hudi.table.action.rollback.BaseRollbackPlanActionExecutor; import org.apache.hudi.table.action.rollback.MergeOnReadRollbackActionExecutor; +import org.apache.hudi.util.CommonClientUtils; import java.util.Collections; import java.util.Iterator; @@ -141,7 +142,7 @@ public HoodieWriteMetadata> logCompact( public Iterator> handleInsertsForLogCompaction(String instantTime, String partitionPath, String fileId, Map> recordMap, Map header) { - HoodieWriteHandle appendHandle = getMetaClient().getTableConfig().isLSMTreeStorageLayout() + HoodieWriteHandle appendHandle = CommonClientUtils.shouldWriteNativeLogs(config, getMetaClient().getTableConfig()) ? new HoodieNativeLogAppendHandle(config, instantTime, this, partitionPath, fileId, recordMap.values().iterator(), taskContextSupplier, header) : new HoodieInlineLogAppendHandle(config, instantTime, this, diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/format/FlinkRecordContext.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/format/FlinkRecordContext.java index e2406b829693e..92c42ece39813 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/format/FlinkRecordContext.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/format/FlinkRecordContext.java @@ -152,6 +152,11 @@ public RowData constructEngineRecord(HoodieSchema recordSchema, Object[] fieldVa return GenericRowData.of(fieldValues); } + @Override + public HoodieRecord.HoodieRecordType getEngineRecordType() { + return HoodieRecord.HoodieRecordType.FLINK; + } + @Override public RowData mergeWithEngineRecord(HoodieSchema schema, Map updateValues, diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java index 76f61f09f7380..3ca5a4515e57a 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java @@ -892,15 +892,15 @@ private void verifyMetadataRecordKeyExcludeFromPayloadLogFiles(HoodieTable table private void verifyMetadataRawRecords(HoodieTable table, List logFiles, boolean enableMetaFields) throws IOException { for (HoodieLogFile logFile : logFiles) { List pathInfoList = storage.listDirectEntries(logFile.getPath()); - HoodieSchema writerSchema = TableSchemaResolver.readSchemaFromLogFile(storage, + HoodieSchema writerSchema = TableSchemaResolver.readSchemaFromLogFile(table.getMetaClient(), logFile.getPath()); if (writerSchema == null) { // not a data block continue; } - try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage, - new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { + try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader( + table.getMetaClient(), new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { while (logFileReader.hasNext()) { HoodieLogBlock logBlock = logFileReader.next(); if (logBlock instanceof HoodieDataBlock) { @@ -2913,7 +2913,7 @@ private void validateMetadata(HoodieJavaWriteClient testClient, Option i verifyMetadataRawRecords(table, logFiles, false); } if (COLUMN_STATS.getPartitionPath().equals(partition)) { - verifyMetadataColumnStatsRecords(logFiles); + verifyMetadataColumnStatsRecords(metadataMetaClient, logFiles); } } catch (IOException e) { log.error("Metadata record validation failed", e); @@ -2926,18 +2926,19 @@ private void validateMetadata(HoodieJavaWriteClient testClient, Option i } } - private void verifyMetadataColumnStatsRecords(List logFiles) throws IOException { + private void verifyMetadataColumnStatsRecords(HoodieTableMetaClient metadataMetaClient, + List logFiles) throws IOException { for (HoodieLogFile logFile : logFiles) { List pathInfoList = storage.listDirectEntries(logFile.getPath()); - HoodieSchema writerSchema = TableSchemaResolver.readSchemaFromLogFile(storage, + HoodieSchema writerSchema = TableSchemaResolver.readSchemaFromLogFile(metadataMetaClient, logFile.getPath()); if (writerSchema == null) { // not a data block continue; } - try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage, - new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { + try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader( + metadataMetaClient, new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { while (logFileReader.hasNext()) { HoodieLogBlock logBlock = logFileReader.next(); if (logBlock instanceof HoodieDataBlock) { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/common/model/HoodieSparkRecord.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/common/model/HoodieSparkRecord.java index a52b711b9be45..1632327e74ca2 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/common/model/HoodieSparkRecord.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/common/model/HoodieSparkRecord.java @@ -357,11 +357,15 @@ protected Comparable doGetOrderingValue(HoodieSchema recordSchema, Properties HoodieInternalRowUtils.getCachedPosList(structType, field); if (cachedNestedFieldPath.isDefined()) { NestedFieldPath nestedFieldPath = cachedNestedFieldPath.get(); + Object value = HoodieUnsafeRowUtils.getNestedInternalRowValue(data, nestedFieldPath); + if (value == null) { + return OrderingValues.getDefault(); + } if (nestedFieldPath.parts()[0]._2.dataType() instanceof org.apache.spark.sql.types.StringType) { return SparkAdapterSupport$.MODULE$.sparkAdapter().getUTF8StringFactory() - .wrapUTF8String((UTF8String) HoodieUnsafeRowUtils.getNestedInternalRowValue(data, nestedFieldPath)); + .wrapUTF8String(((UTF8String) value).copy()); } - return (Comparable) HoodieUnsafeRowUtils.getNestedInternalRowValue(data, nestedFieldPath); + return (Comparable) value; } return OrderingValues.getDefault(); }); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriter.java index 311ba058eccb2..434b9ddcb7b16 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriter.java @@ -28,15 +28,13 @@ import java.io.IOException; import java.util.Properties; -public interface HoodieSparkFileWriter extends HoodieFileWriter { +public interface HoodieSparkFileWriter extends HoodieFileWriter { boolean canWrite(); void close() throws IOException; void writeRowWithMetadata(HoodieKey recordKey, InternalRow row) throws IOException; - void writeRow(String recordKey, InternalRow row) throws IOException; - @Override default void write(String recordKey, HoodieRecord record, HoodieSchema schema, Properties props) throws IOException { writeRow(recordKey, (InternalRow) record.getData()); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java index f1af27db0a584..f7c09de9686a0 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java @@ -52,10 +52,10 @@ import org.apache.spark.sql.catalyst.expressions.UnsafeProjection; import org.apache.spark.sql.catalyst.expressions.UnsafeRow; import org.apache.spark.sql.catalyst.util.RebaseDateTime; +import org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormatHelper$; import org.apache.spark.sql.execution.datasources.parquet.HoodieParquetReadSupport; import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters; import org.apache.spark.sql.execution.datasources.parquet.ParquetReadSupport; -import org.apache.spark.sql.execution.datasources.parquet.ParquetToSparkSchemaConverter; import org.apache.spark.sql.execution.datasources.parquet.SparkBasicSchemaEvolution; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; @@ -158,7 +158,8 @@ public ClosableIterator getUnsafeRowIterator(HoodieSchema requestedSc Option messageSchema = Option.of(getAvroSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(nonNullSchema)); boolean enableTimestampFieldRepair = storage.getConf().getBoolean(ENABLE_LOGICAL_TIMESTAMP_REPAIR, true); - StructType dataStructType = convertToStruct(enableTimestampFieldRepair ? SchemaRepair.repairLogicalTypes(getFileSchema(), messageSchema) : getFileSchema()); + MessageType dataMessageType = enableTimestampFieldRepair ? SchemaRepair.repairLogicalTypes(getFileSchema(), messageSchema) : getFileSchema(); + StructType dataStructType = convertToStruct(dataMessageType); SparkBasicSchemaEvolution evolution = new SparkBasicSchemaEvolution(dataStructType, readStructSchema, SQLConf.get().sessionLocalTimeZone()); String readSchemaJson = evolution.getRequestSchema().json(); SQLConf sqlConf = SQLConf.get(); @@ -248,7 +249,8 @@ protected StructType getStructSchema() { } private StructType convertToStruct(MessageType messageType) { - return new ParquetToSparkSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(messageType); + return HoodieParquetFileFormatHelper$.MODULE$.convertParquetSchemaToSparkSchema( + storage.getConf().unwrapAs(Configuration.class), messageType); } @Override diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkMergeOnReadTable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkMergeOnReadTable.java index dc98e5480b2fd..f6aa9c8a53dd2 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkMergeOnReadTable.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkMergeOnReadTable.java @@ -60,6 +60,7 @@ import org.apache.hudi.table.action.rollback.BaseRollbackPlanActionExecutor; import org.apache.hudi.table.action.rollback.MergeOnReadRollbackActionExecutor; import org.apache.hudi.table.action.rollback.RestorePlanActionExecutor; +import org.apache.hudi.util.CommonClientUtils; import lombok.extern.slf4j.Slf4j; @@ -202,7 +203,7 @@ public Option scheduleRollback(HoodieEngineContext context, public Iterator> handleInsertsForLogCompaction(String instantTime, String partitionPath, String fileId, Map> recordMap, Map header) { - HoodieWriteHandle appendHandle = getMetaClient().getTableConfig().isLSMTreeStorageLayout() + HoodieWriteHandle appendHandle = CommonClientUtils.shouldWriteNativeLogs(config, getMetaClient().getTableConfig()) ? new HoodieNativeLogAppendHandle(config, instantTime, this, partitionPath, fileId, recordMap.values().iterator(), taskContextSupplier, header) : new HoodieInlineLogAppendHandle(config, instantTime, this, diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRecordContext.java b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRecordContext.java index 6bd5420af014f..e8bacf6d3b78a 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRecordContext.java +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRecordContext.java @@ -136,6 +136,11 @@ public InternalRow constructEngineRecord(HoodieSchema recordSchema, Object[] fie return new GenericInternalRow(fieldValues); } + @Override + public HoodieRecord.HoodieRecordType getEngineRecordType() { + return HoodieRecord.HoodieRecordType.SPARK; + } + @Override public InternalRow mergeWithEngineRecord(HoodieSchema schema, Map updateValues, @@ -156,11 +161,12 @@ public InternalRow mergeWithEngineRecord(HoodieSchema schema, @Override public Comparable convertValueToEngineType(Comparable value) { if (value instanceof String) { - // Spark reads String field values as UTF8String. - // To foster value comparison, if the value is of String type, e.g., from - // the delete record, we convert it to UTF8String type. - // [SPARK-46832] UTF8String doesn't support compareTo anymore - return SparkAdapterSupport$.MODULE$.sparkAdapter().getUTF8StringFactory().wrapUTF8String(UTF8String.fromString((String) value)); + // Spark represents String field values as UTF8String in InternalRow. + // NOTE: the returned raw UTF8String is meant for storage in an engine record; it does + // not support compareTo since [SPARK-46832]. Callers that need comparison must go + // through convertOrderingValueToEngineType/ensureComparability, which wrap the value + // into HoodieUTF8String. + return UTF8String.fromString((String) value); } return value; } @@ -177,7 +183,7 @@ protected Comparable ensureComparability(Object value) { // the delete record, we convert it to UTF8String type. // [SPARK-46832] UTF8String doesn't support compareTo anymore if (value instanceof UTF8String) { - return SparkAdapterSupport$.MODULE$.sparkAdapter().getUTF8StringFactory().wrapUTF8String((UTF8String) value); + return SparkAdapterSupport$.MODULE$.sparkAdapter().getUTF8StringFactory().wrapUTF8String(((UTF8String) value).copy()); } return (Comparable) value; } diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala index 06579bd99d27a..bfeb5df2848b3 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala @@ -160,10 +160,31 @@ class SparkFileFormatInternalRowReaderContext(baseFileReader: SparkColumnarFileR val (readSchema, readFilters) = getSchemaAndFiltersForRead(parquetReadStructType, hasRowIndexField) if (FSUtils.isLogFile(filePath)) { // NOTE: now only primary key based filtering is supported for log files + // Position-based merging pairs log records with the RECORD_POSITIONS bitmap by index (see + // PositionBasedFileGroupRecordBuffer), which requires the record stream to contain every + // record of the block. Pushing filters into the scan can physically drop records (e.g. with + // parquet record-level filtering) and misalign that pairing, so push no filters in that case. + val logReadFilters = if (getShouldMergeUseRecordPosition) Seq.empty[Filter] else readFilters // Variant alignment happens later via getLogBlockRecordProjection in the merge buffer. - new HoodieSparkFileReaderFactory(storage).newParquetFileReader(filePath) - .asInstanceOf[HoodieSparkParquetReader].getUnsafeRowIterator(requiredSchema, readFilters.asJava) - .asInstanceOf[ClosableIterator[InternalRow]] + // Log files reach this method either as an inline parquet data block or as a native log file. + // Inline paths do not carry the parquet extension, so resolve them by the log block contract. + val fileFormat = if (FSUtils.isInlineLogFile(filePath.getName)) { + HoodieFileFormat.PARQUET + } else { + HoodieFileFormat.fromFileExtension(filePath.getFileExtension) + } + fileFormat match { + case HoodieFileFormat.PARQUET => + new HoodieSparkFileReaderFactory(storage).newParquetFileReader(filePath) + .asInstanceOf[HoodieSparkParquetReader].getUnsafeRowIterator(requiredSchema, logReadFilters.asJava) + .asInstanceOf[ClosableIterator[InternalRow]] + case _ => + val fileInfo = sparkAdapter.getSparkPartitionedFileUtils + .createPartitionedFile(InternalRow.empty, filePath, start, length) + new CloseableInternalRowIterator(baseFileReader.read(fileInfo, + readSchema, StructType(Seq.empty), getSchemaHandler.getInternalSchemaOpt, + logReadFilters, storage.getConf.asInstanceOf[StorageConfiguration[Configuration]])) + } } else { // partition value is empty because the spark parquet reader will append the partition columns to // each row if they are given. That is the only usage of the partition values in the reader. diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieParquetFileFormatHelper.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieParquetFileFormatHelper.scala index 6c2133261e779..195894e194087 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieParquetFileFormatHelper.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieParquetFileFormatHelper.scala @@ -37,19 +37,23 @@ object HoodieParquetFileFormatHelper { requiredSchema: StructType): (java.util.Map[Integer, Pair[DataType, DataType]], StructType) = { val originalSchema = parquetFileMetaData.getSchema + val fileStruct = convertParquetSchemaToSparkSchema(hadoopConf, originalSchema) + SparkSchemaTransformUtils.buildImplicitSchemaChangeInfo(fileStruct, requiredSchema) + } + + def convertParquetSchemaToSparkSchema(hadoopConf: Configuration, schema: MessageType): StructType = { // Spark 3.3's ParquetToSparkSchemaConverter throws "Illegal Parquet type: FIXED_LEN_BYTE_ARRAY" // for unannotated FLBA columns (e.g., Hudi VECTOR type). This was fixed in Spark 3.4 // (SPARK-41096 / https://github.com/apache/spark/pull/38628) which maps bare FLBA to BinaryType. // On Spark 3.3 only, rewrite bare FLBA to BINARY before conversion. val safeSchema = if (!HoodieSparkUtils.gteqSpark3_4) { - rewriteFixedLenByteArrayToBinary(originalSchema) + rewriteFixedLenByteArrayToBinary(schema) } else { - originalSchema + schema } val convert = new ParquetToSparkSchemaConverter(hadoopConf) - val fileStruct = convert.convert(safeSchema) - SparkSchemaTransformUtils.buildImplicitSchemaChangeInfo(fileStruct, requiredSchema) + convert.convert(safeSchema) } /** diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java index 96ddb745f986e..0ad9b95e00972 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java @@ -520,14 +520,14 @@ private void verifyMetadataRawRecords(HoodieTable table, List log for (HoodieLogFile logFile : logFiles) { List pathInfoList = storage.listDirectEntries(logFile.getPath()); HoodieSchema writerSchema = - TableSchemaResolver.readSchemaFromLogFile(storage, logFile.getPath()); + TableSchemaResolver.readSchemaFromLogFile(table.getMetaClient(), logFile.getPath()); if (writerSchema == null) { // not a data block continue; } - try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage, - new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { + try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader( + table.getMetaClient(), new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { while (logFileReader.hasNext()) { HoodieLogBlock logBlock = logFileReader.next(); if (logBlock instanceof HoodieDataBlock) { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java index fb001dd4e2856..83cac12c0fd60 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java @@ -165,13 +165,21 @@ public void init() throws Exception { } public void init(HoodieTableType tableType) throws Exception { + init(tableType, null); + } + + private void init(HoodieTableType tableType, HoodieTableVersion tableVersion) throws Exception { initPath(); initSparkContexts(); initTimelineService(); initMetaClient(); storage = metaClient.getStorage(); metaClient.getStorage().createDirectory(new StoragePath(basePath)); - metaClient = HoodieTestUtils.init(storageConf, basePath, tableType); + Properties properties = new Properties(); + if (tableVersion != null) { + properties.setProperty(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), String.valueOf(tableVersion.versionCode())); + } + metaClient = HoodieTestUtils.init(storageConf, basePath, tableType, properties); } private void initWriteConfigAndMetatableWriter(HoodieWriteConfig writeConfig, boolean enableMetadataTable) { @@ -218,6 +226,18 @@ private HoodieWriteConfig initTestTableAndGetWriteConfig(boolean enableMetadata, HoodieFailedWritesCleaningPolicy.EAGER, WriteConcurrencyMode.SINGLE_WRITER); } + private HoodieWriteConfig initTestTableAndGetWriteConfig(boolean enableMetadata, + int minArchivalCommits, + int maxArchivalCommits, + int maxDeltaCommitsMetadataTable, + HoodieTableType tableType, + HoodieTableVersion tableVersion) throws Exception { + return initTestTableAndGetWriteConfig(enableMetadata, minArchivalCommits, maxArchivalCommits, + 5, maxDeltaCommitsMetadataTable, tableType, 10, + HoodieFailedWritesCleaningPolicy.EAGER, WriteConcurrencyMode.SINGLE_WRITER, + ARCHIVE_BEYOND_SAVEPOINT.defaultValue(), tableVersion); + } + private HoodieWriteConfig initTestTableAndGetWriteConfig(boolean enableMetadata, int minArchivalCommits, int maxArchivalCommits, @@ -260,8 +280,24 @@ private HoodieWriteConfig initTestTableAndGetWriteConfig(boolean enableMetadata, HoodieFailedWritesCleaningPolicy failedWritesCleaningPolicy, WriteConcurrencyMode writeConcurrencyMode, boolean archiveProceedBeyondSavepoints) throws Exception { - init(tableType); - HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(basePath) + return initTestTableAndGetWriteConfig(enableMetadata, minArchivalCommits, maxArchivalCommits, maxDeltaCommits, + maxDeltaCommitsMetadataTable, tableType, archiveFilesBatch, failedWritesCleaningPolicy, writeConcurrencyMode, + archiveProceedBeyondSavepoints, null); + } + + private HoodieWriteConfig initTestTableAndGetWriteConfig(boolean enableMetadata, + int minArchivalCommits, + int maxArchivalCommits, + int maxDeltaCommits, + int maxDeltaCommitsMetadataTable, + HoodieTableType tableType, + int archiveFilesBatch, + HoodieFailedWritesCleaningPolicy failedWritesCleaningPolicy, + WriteConcurrencyMode writeConcurrencyMode, + boolean archiveProceedBeyondSavepoints, + HoodieTableVersion tableVersion) throws Exception { + init(tableType, tableVersion); + HoodieWriteConfig.Builder writeConfigBuilder = HoodieWriteConfig.newBuilder().withPath(basePath) .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2) .withCleanConfig(HoodieCleanConfig.newBuilder().retainCommits(1).withFailedWritesCleaningPolicy(failedWritesCleaningPolicy).build()) .withArchivalConfig(HoodieArchivalConfig.newBuilder() @@ -279,7 +315,11 @@ private HoodieWriteConfig initTestTableAndGetWriteConfig(boolean enableMetadata, .withWriteConcurrencyMode(writeConcurrencyMode) .withLockConfig(HoodieLockConfig.newBuilder().withLockProvider(InProcessLockProvider.class) .build()) - .forTable("test-trip-table").build(); + .forTable("test-trip-table"); + if (tableVersion != null) { + writeConfigBuilder.withWriteTableVersion(tableVersion.versionCode()); + } + HoodieWriteConfig writeConfig = writeConfigBuilder.build(); initWriteConfigAndMetatableWriter(writeConfig, enableMetadata); return writeConfig; } @@ -788,7 +828,8 @@ public void testReadArchivedCompactionPlan() throws Exception { @Test public void testDowngradeArchivedTimeline() throws Exception { - HoodieWriteConfig writeConfig = initTestTableAndGetWriteConfig(false, 1, 2, 5, HoodieTableType.MERGE_ON_READ); + // downgrade from 10 to 9 is not supported yet. + HoodieWriteConfig writeConfig = initTestTableAndGetWriteConfig(false, 1, 2, 5, HoodieTableType.MERGE_ON_READ, HoodieTableVersion.NINE); // do ingestion and trigger archive actions here. Map cleanStats = new HashMap<>(); @@ -2397,7 +2438,7 @@ public void testArchivalBlocksOnCleanECTRWithTimelineArchiverV2AndVersion9() thr addCleanCommitWithECTR(testTable, "00000006", "00000003", "00000005"); metaClient = HoodieTableMetaClient.reload(metaClient); - assertEquals(HoodieTableVersion.NINE, metaClient.getTableConfig().getTableVersion(), + assertEquals(HoodieTableVersion.TEN, metaClient.getTableConfig().getTableVersion(), "Table should be version 9"); // When: trigger archival using TimelineArchiverV2 diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java index 9808de708374d..6d0346a2c4de1 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java @@ -45,6 +45,7 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -60,7 +61,7 @@ public void testDeleteFailureDuringMarkerReconciliation(DeleteFailureType failur HoodieStorage localStorage = HoodieStorageUtils.getStorage(basePath, CONF); WriteMarkers writeMarkers = mock(WriteMarkers.class); String partitionPath = "p1"; - List datafiles = Arrays.asList("file1", "file2", "file3"); + List datafiles = Arrays.asList("file1", "file2", "file3", ".file_1.log.parquet", ".file_1.deletes.parquet"); List writeStatList = new ArrayList<>(); Set markerList = new HashSet<>(); datafiles.forEach(fileName -> { @@ -125,6 +126,12 @@ public void testDeleteFailureDuringMarkerReconciliation(DeleteFailureType failur throw new HoodieException("Failed to validate that file exists " + fileName); } }); + try { + verify(storage, never()).deleteFile(new StoragePath(basePath + "/" + partitionPath + "/.file_1.log.parquet")); + verify(storage, never()).deleteFile(new StoragePath(basePath + "/" + partitionPath + "/.file_1.deletes.parquet")); + } catch (IOException e) { + throw new HoodieException("Failed to validate native log files were not deleted", e); + } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java index fceeeaf84b97a..38652748fb70f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java @@ -149,6 +149,11 @@ public IndexedRecord constructEngineRecord(HoodieSchema recordSchema, Object[] f return record; } + @Override + public HoodieRecord.HoodieRecordType getEngineRecordType() { + return HoodieRecord.HoodieRecordType.AVRO; + } + @Override public IndexedRecord mergeWithEngineRecord(HoodieSchema schema, Map updateValues, diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java index 3b1dd129fbd1d..7e42ffd780a75 100644 --- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java @@ -47,6 +47,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.core.io.storage.HoodieAvroFileReader; import org.apache.hudi.core.io.storage.HoodieIOFactory; +import org.apache.hudi.core.io.storage.HoodieSeekingFileReader; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StorageConfiguration; @@ -188,6 +189,29 @@ public ClosableIterator getFileRecordIterator( return getFileRecordIterator(reader, filePath, isLogFile, dataSchema, requiredSchema); } + @Override + public ClosableIterator lookupRecords( + StoragePath filePath, + HoodieFileFormat fileFormat, + HoodieSchema readerSchema, + HoodieStorage storage, + List keys, + boolean fullKey) throws IOException { + try (HoodieAvroFileReader avroFileReader = (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(storage) + .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO) + .getFileReader(hoodieReaderConfig, filePath, fileFormat, Option.empty())) { + if (avroFileReader instanceof HoodieSeekingFileReader) { + HoodieSeekingFileReader seekingFileReader = (HoodieSeekingFileReader) avroFileReader; + if (fullKey && avroFileReader.supportKeyPredicate()) { + return seekingFileReader.getEngineRecordsByKeysIterator(keys, readerSchema); + } else if (!fullKey && avroFileReader.supportKeyPrefixPredicate()) { + return seekingFileReader.getEngineRecordsByKeyPrefixIterator(keys, readerSchema); + } + } + } + return super.lookupRecords(filePath, fileFormat, readerSchema, storage, keys, fullKey); + } + private HoodieAvroFileReader getOrCreateFileReader( StoragePath path, boolean isLogFile, Function func) throws IOException { if (reusableFileReaders.containsKey(path)) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java index d20db5579bd42..da51a18136784 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.expression.Predicate; +import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordMerger; @@ -58,6 +59,7 @@ import lombok.experimental.Accessors; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -210,6 +212,35 @@ public ClosableIterator getFileRecordIterator( return getFileRecordIterator(storagePathInfo.getPath(), start, length, dataSchema, requiredSchema, storage); } + /** + * Optionally returns an iterator over records matching the given keys directly from the file. + * + *

Reader contexts can override this when the underlying engine-specific reader supports key + * seeking. The default implementation scans the file and filters records by key. + */ + public ClosableIterator lookupRecords( + StoragePath filePath, + HoodieFileFormat fileFormat, + HoodieSchema readerSchema, + HoodieStorage storage, + List keys, + boolean fullKey) throws IOException { + ClosableIterator fileRecordIterator = getFileRecordIterator( + filePath, 0, FSUtils.getFileSize(storage, filePath), readerSchema, readerSchema, storage); + if (keys.isEmpty()) { + return fileRecordIterator; + } + + HashSet keySet = new HashSet<>(keys); + return new CloseableFilterIterator<>(fileRecordIterator, record -> { + String recordKey = recordContext.getRecordKey(record, readerSchema); + if (recordKey == null) { + throw new IllegalStateException(String.format("Record without a key (%s)", record)); + } + return fullKey ? keySet.contains(recordKey) : keys.stream().anyMatch(recordKey::startsWith); + }); + } + /** * @param mergeMode record merge mode * @param mergeStrategyId record merge strategy ID diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java index d63c9bd63408b..81cd60e8ec8d5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/RecordContext.java @@ -170,6 +170,14 @@ public abstract T mergeWithEngineRecord(HoodieSchema schema, */ public abstract T constructEngineRecord(HoodieSchema recordSchema, Object[] fieldValues); + /** + * Returns the engine record type produced by this context (e.g. {@code AVRO}, {@code SPARK}, {@code FLINK}, + * {@code HIVE}). This mirrors the concrete type of records built by {@link #constructEngineRecord}, so callers that + * feed those engine records into a {@link org.apache.hudi.io.storage.HoodieFileWriter} can pick a writer that + * matches. Used by the native log delete writer, where the record object type and the writer must agree. + */ + public abstract HoodieRecord.HoodieRecordType getEngineRecordType(); + /** * Gets the record key in String. * @@ -202,16 +210,18 @@ public String getRecordKey(T record, HoodieSchema schema) { public abstract String getMetaFieldValue(T record, int pos); /** - * Returns the value to a type representation in a specific engine. + * Converts the value to the engine-native type representation that can be stored directly + * as a field value inside an engine record, e.g., {@code UTF8String} for a {@link String} + * value in Spark. *

- * This can be overridden by the reader context implementation on a specific engine to handle - * engine-specific field type system. For example, Spark uses {@code UTF8String} to represent - * {@link String} field values, so we need to convert the values to {@code UTF8String} type - * in Spark for proper value comparison. + * NOTE: The returned value is NOT guaranteed to support comparison on every engine + * (e.g., Spark's {@code UTF8String} no longer supports {@code compareTo}, see SPARK-46832). + * Callers that need a comparable value, e.g., for ordering, should use + * {@link #convertOrderingValueToEngineType} or {@link #ensureComparability} instead. * * @param value {@link Comparable} value to be converted. * - * @return the converted value in a type representation in a specific engine. + * @return the converted value in the engine-native type representation. */ public Comparable convertValueToEngineType(Comparable value) { return value; @@ -222,12 +232,14 @@ public Comparable convertPartitionValueToEngineType(Comparable value) { } /** - * Converts the ordering value to the specific engine type. + * Converts the ordering value to the specific engine type, guaranteeing the returned + * value supports comparison, e.g., by wrapping engine types that do not implement + * {@code compareTo} (see {@link #ensureComparability}). */ public Comparable convertOrderingValueToEngineType(Comparable value) { return value instanceof ArrayComparable - ? ((ArrayComparable) value).apply(this::convertValueToEngineType) - : convertValueToEngineType(value); + ? ((ArrayComparable) value).apply(val -> ensureComparability(convertValueToEngineType(val))) + : ensureComparability(convertValueToEngineType(value)); } /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index ff39c9f23358d..5aff1b9c4ca65 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -356,6 +356,10 @@ public static String getFileIdFromFileName(String fileName) { } public static String getFileIdFromLogPath(StoragePath path) { + Option nativeLogMatcher = matchNativeLogFile(path.getName()); + if (nativeLogMatcher.isPresent()) { + return nativeLogMatcher.get().group(1); + } Option logFileMatcher = matchLogFile(path.getName()); if (!logFileMatcher.isPresent()) { throw new InvalidHoodiePathException(path, "LogFile"); @@ -484,17 +488,21 @@ public static String makeNativeLogFileName(String fileId, String writeToken, Str return String.format("%s_%s_%s_%d.%s.%s", fileId, writeToken, deltaCommitTime, version, extension, formatSuffix); } - public static boolean isBaseFile(StoragePath path) { - if (matchNativeLogFile(path.getName()).isPresent()) { + public static boolean isBaseFile(String path) { + if (matchNativeLogFile(path).isPresent()) { return false; } - String extension = getFileExtension(path.getName()); + String extension = getFileExtension(path); if (HoodieFileFormat.BASE_FILE_EXTENSIONS.contains(extension)) { - return BASE_FILE_PATTERN.matcher(path.getName()).matches(); + return BASE_FILE_PATTERN.matcher(path).matches(); } return false; } + public static boolean isBaseFile(StoragePath path) { + return isBaseFile(path.getName()); + } + public static String getWriteTokenFromBaseFile(String fileName) { Matcher matcher = BASE_FILE_PATTERN.matcher(fileName); if (!matcher.find()) { @@ -521,6 +529,10 @@ public static boolean isLogFile(String fileName) { return false; } + public static boolean isInlineLogFile(String fileName) { + return matchNativeLogFile(fileName).isEmpty(); + } + public static Option matchNativeLogFile(String fileName) { if (StringUtils.isNullOrEmpty(fileName)) { return Option.empty(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemas.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemas.java index 3a2ec7015f5f0..917988dd8aa4e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemas.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemas.java @@ -33,11 +33,17 @@ public class HoodieSchemas { public static HoodieSchema createDeleteLogSchema(HoodieSchema tableSchema, List orderingFieldNames) { + // Native delete logs store only the record key plus optional ordering values, so ordering fields in + // the delete-log schema must always be nullable even when the table schema marks them required. + // A delete record such as HoodieEmptyRecord may carry OrderingValues.getDefault() as an in-memory + // sentinel rather than a real field value. Persist NULL for that missing value so readers can map it + // back to the default ordering without confusing it with a real business value such as 0. List fields = Stream.concat( Stream.of(createNewSchemaField( HoodieRecord.RECORD_KEY_METADATA_FIELD, HoodieSchema.create(HoodieSchemaType.STRING), null, null)), orderingFieldNames.stream().map(orderingFieldName -> tableSchema.getField(orderingFieldName) - .map(HoodieSchemaUtils::createNewSchemaField) + .map(field -> createNewSchemaField( + field.name(), HoodieSchema.createNullable(field.schema()), field.doc().orElse(null), HoodieSchema.NULL_VALUE)) .orElseThrow(() -> new IllegalArgumentException("Ordering field " + orderingFieldName + " not found in table schema")))) .collect(Collectors.toList()); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java index 25e79efcf4ba9..e5b41dbc1c596 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java @@ -70,7 +70,7 @@ public enum HoodieTableVersion { private final TimelineLayoutVersion timelineLayoutVersion; public static HoodieTableVersion current() { - return NINE; + return TEN; } public static HoodieTableVersion fromVersionCode(int versionCode) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java index 031a8490caac3..3888ca20456fd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java @@ -46,7 +46,6 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieSchemaNotFoundException; import org.apache.hudi.exception.InvalidTableException; -import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.util.Lazy; @@ -279,7 +278,7 @@ public Option getTableSchemaFromLatestCommit(boolean includeMetada } private HoodieSchema readSchemaFromLogFile(StoragePath path) throws IOException { - return readSchemaFromLogFile(metaClient.getRawStorage(), path); + return readSchemaFromLogFile(metaClient, path); } /** @@ -287,16 +286,16 @@ private HoodieSchema readSchemaFromLogFile(StoragePath path) throws IOException * * @return */ - public static HoodieSchema readSchemaFromLogFile(HoodieStorage storage, StoragePath path) throws IOException { + public static HoodieSchema readSchemaFromLogFile(HoodieTableMetaClient metaClient, StoragePath path) throws IOException { Option nativeLogMatcherOpt = FSUtils.matchNativeLogFile(path.getName()); if (nativeLogMatcherOpt.isPresent()) { - return NativeLogFooterMetadata.readSchemaFromNativeLogFile(storage, path, nativeLogMatcherOpt.get()); + return NativeLogFooterMetadata.readSchemaFromNativeLogFile(metaClient.getStorage(), path, nativeLogMatcherOpt.get()); } // We only need to read the schema from the log block header, // so we read the block lazily to avoid reading block content // containing the records - try (Reader reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(path), null, false)) { + try (Reader reader = HoodieLogFormat.newReader(metaClient, new HoodieLogFile(path), null, false)) { HoodieDataBlock lastBlock = null; while (reader.hasNext()) { HoodieLogBlock block = reader.next(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java index 930e336cdd2ed..777716a383add 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java @@ -258,7 +258,7 @@ protected final synchronized void scanInternal(Option keySpecOpt, boole HoodieLogFormatReader logFormatReaderWrapper = null; try { // Iterate over the paths - logFormatReaderWrapper = new HoodieLogFormatReader(storage, + logFormatReaderWrapper = new HoodieLogFormatReader(storage, hoodieTableMetaClient, logFilePaths.stream() .map(logFile -> new HoodieLogFile(new StoragePath(logFile))) .collect(Collectors.toList()), diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java index f2460fae0d066..170590df8829c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java @@ -226,7 +226,7 @@ protected final synchronized void scanInternal(Option keySpecOpt, boole HoodieLogFormatReader logFormatReaderWrapper = null; try { // Iterate over the paths - logFormatReaderWrapper = new HoodieLogFormatReader(storage, readerContext, hoodieTableMetaClient, logFiles, + logFormatReaderWrapper = new HoodieLogFormatReader(storage, hoodieTableMetaClient, logFiles, readerSchema, reverseReader, bufferSize, shouldLookupRecords(), recordKeyField, internalSchema); /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java index 45ff2624dc816..e338e32a62833 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java @@ -21,8 +21,10 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; @@ -240,12 +242,24 @@ interface Reader extends Closeable, Iterator { HoodieLogBlock prev() throws IOException; } - static HoodieLogFormat.Reader newReader(HoodieStorage storage, HoodieLogFile logFile, HoodieSchema readerSchema) - throws IOException { - return new HoodieLogFileReader(storage, logFile, readerSchema, HoodieLogFileReader.DEFAULT_BUFFER_SIZE); + static HoodieLogFormat.Reader newReader(HoodieTableMetaClient metaClient, HoodieLogFile logFile, + HoodieSchema readerSchema) throws IOException { + return newReader(metaClient, logFile, readerSchema, false); } - static HoodieLogFormat.Reader newReader(HoodieStorage storage, HoodieLogFile logFile, HoodieSchema readerSchema, boolean reverseReader) throws IOException { + static HoodieLogFormat.Reader newReader(HoodieTableMetaClient metaClient, HoodieLogFile logFile, + HoodieSchema readerSchema, boolean reverseReader) throws IOException { + HoodieStorage storage = metaClient.getStorage(); + if (FSUtils.matchNativeLogFile(logFile.getFileName()).isPresent()) { + StoragePath logFileParent = logFile.getPath().getParent(); + return new HoodieNativeLogFileReader( + storage, + logFile, + readerSchema, + HoodieRecordUtils.getOrderingFieldNames(metaClient.getTableConfig().getRecordMergeMode(), metaClient), + FSUtils.getRelativePartitionPath(metaClient.getBasePath(), logFileParent), + metaClient.getTableConfig().getProps()); + } return new HoodieLogFileReader(storage, logFile, readerSchema, HoodieLogFileReader.DEFAULT_BUFFER_SIZE, reverseReader); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java index c4fa4d6c012cd..e9fd5f2afc189 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java @@ -18,7 +18,6 @@ package org.apache.hudi.common.table.log; -import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.schema.HoodieSchema; @@ -29,6 +28,7 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import lombok.extern.slf4j.Slf4j; @@ -46,26 +46,16 @@ public class HoodieLogFormatReader implements HoodieLogFormat.Reader { private final HoodieStorage storage; private final HoodieSchema readerSchema; private final InternalSchema internalSchema; - private final HoodieReaderContext readerContext; private final HoodieTableMetaClient metaClient; private final String recordKeyField; private final boolean enableInlineReading; private final int bufferSize; - HoodieLogFormatReader(HoodieStorage storage, List logFiles, HoodieSchema readerSchema, - boolean reverseLogReader, int bufferSize, boolean enableRecordLookups, - String recordKeyField, InternalSchema internalSchema) throws IOException { - this(storage, null, null, logFiles, readerSchema, reverseLogReader, bufferSize, enableRecordLookups, - recordKeyField, internalSchema); - } - - HoodieLogFormatReader(HoodieStorage storage, HoodieReaderContext readerContext, HoodieTableMetaClient metaClient, - List logFiles, HoodieSchema readerSchema, - boolean reverseLogReader, int bufferSize, boolean enableRecordLookups, + HoodieLogFormatReader(HoodieStorage storage, HoodieTableMetaClient metaClient, List logFiles, + HoodieSchema readerSchema, boolean reverseLogReader, int bufferSize, boolean enableRecordLookups, String recordKeyField, InternalSchema internalSchema) throws IOException { this.logFiles = logFiles; this.storage = storage; - this.readerContext = readerContext; this.metaClient = metaClient; this.readerSchema = readerSchema; this.bufferSize = bufferSize; @@ -136,13 +126,21 @@ public HoodieLogBlock prev() throws IOException { private HoodieLogFormat.Reader createReader(HoodieLogFile logFile, boolean reverseLogReader) throws IOException { if (FSUtils.matchNativeLogFile(logFile.getFileName()).isPresent()) { - if (readerContext == null || metaClient == null) { - throw new HoodieNotSupportedException("Native log files require HoodieFileGroupReader based reader context"); + if (metaClient == null) { + throw new HoodieNotSupportedException("Native log files require HoodieTableMetaClient"); } - return new HoodieNativeLogFileReader(storage, logFile, readerContext, readerSchema, - HoodieRecordUtils.getOrderingFieldNames(readerContext.getMergeMode(), metaClient)); + List orderingFieldNames = + HoodieRecordUtils.getOrderingFieldNames(metaClient.getTableConfig().getRecordMergeMode(), metaClient); + return new HoodieNativeLogFileReader(storage, logFile, readerSchema, internalSchema, + orderingFieldNames, + getRelativePartitionPath(logFile), metaClient.getTableConfig().getProps()); } return new HoodieLogFileReader(storage, logFile, readerSchema, bufferSize, reverseLogReader, enableInlineReading, recordKeyField, internalSchema); } + + private String getRelativePartitionPath(HoodieLogFile logFile) { + StoragePath logFileParent = logFile.getPath().getParent(); + return FSUtils.getRelativePartitionPath(metaClient.getBasePath(), logFileParent); + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieNativeLogFileReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieNativeLogFileReader.java index 6f0349811896d..52b76e92dcb11 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieNativeLogFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieNativeLogFileReader.java @@ -19,16 +19,15 @@ package org.apache.hudi.common.table.log; -import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.schema.HoodieSchema; -import org.apache.hudi.common.schema.HoodieSchemas; +import org.apache.hudi.common.schema.internal.InternalSchema; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import org.apache.hudi.common.table.log.block.HoodieNativeDataBlock; -import org.apache.hudi.common.table.log.block.HoodieNativeDeleteBlock; +import org.apache.hudi.common.table.log.block.HoodieNativeLogDataBlock; +import org.apache.hudi.common.table.log.block.HoodieNativeLogDeleteBlock; import org.apache.hudi.common.util.Option; import org.apache.hudi.core.io.storage.HoodieIOFactory; import org.apache.hudi.exception.HoodieIOException; @@ -39,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Properties; /** * Reader for native log files. @@ -47,18 +47,29 @@ public class HoodieNativeLogFileReader implements HoodieLogFormat.Reader { private final HoodieStorage storage; private final HoodieLogFile logFile; - private final HoodieReaderContext readerContext; private final HoodieSchema readerSchema; + private final InternalSchema internalSchema; private final List orderingFieldNames; + private final String partitionPath; + private final Properties props; private boolean consumed; - HoodieNativeLogFileReader(HoodieStorage storage, HoodieLogFile logFile, HoodieReaderContext readerContext, - HoodieSchema readerSchema, List orderingFieldNames) { + HoodieNativeLogFileReader(HoodieStorage storage, HoodieLogFile logFile, + HoodieSchema readerSchema, + List orderingFieldNames, String partitionPath, Properties props) { + this(storage, logFile, readerSchema, null, orderingFieldNames, partitionPath, props); + } + + HoodieNativeLogFileReader(HoodieStorage storage, HoodieLogFile logFile, + HoodieSchema readerSchema, InternalSchema internalSchema, + List orderingFieldNames, String partitionPath, Properties props) { this.storage = storage; this.logFile = logFile; - this.readerContext = readerContext; this.readerSchema = readerSchema; + this.internalSchema = internalSchema == null ? InternalSchema.getEmptyInternalSchema() : internalSchema; this.orderingFieldNames = orderingFieldNames; + this.partitionPath = partitionPath; + this.props = props; } @Override @@ -94,14 +105,13 @@ public HoodieLogBlock next() { consumed = true; HoodieFileFormat fileFormat = getNativeFileFormat(); Map header = getLogBlockHeader(fileFormat); + validateDataBlockHeader(logFile, header); if (FSUtils.isNativeDeleteLogFile(logFile.getFileName())) { - HoodieSchema deleteLogSchema = HoodieSchemas.createDeleteLogSchema( - readerContext.getSchemaHandler().getTableSchema(), orderingFieldNames); - return new HoodieNativeDeleteBlock(storage, logFile, readerContext, deleteLogSchema, - orderingFieldNames, header, new HashMap<>()); + return new HoodieNativeLogDeleteBlock(storage, logFile, + orderingFieldNames, partitionPath, props, header, new HashMap<>()); } - validateDataBlockHeader(logFile, header); - return new HoodieNativeDataBlock(storage, logFile, fileFormat, Option.ofNullable(readerSchema), header, new HashMap<>()); + Option readerSchemaOpt = internalSchema.isEmptySchema() ? Option.ofNullable(readerSchema) : Option.empty(); + return new HoodieNativeLogDataBlock(storage, logFile, fileFormat, readerSchemaOpt, header, new HashMap<>()); } @Override @@ -122,7 +132,7 @@ static void validateDataBlockHeader(HoodieLogFile logFile, Map> getAllLogFilesWithMaxCommit(Hoodi Map> logFilesWithMaxCommit = engineContext.mapToPair(logFiles, logFile -> { // read all blocks within the log file and find the commit associated with each log block List blocksWithinLogFile = new ArrayList<>(); - try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(metaClient.getStorage(), logFile, null)) { + try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(metaClient, logFile, null)) { while (reader.hasNext()) { HoodieLogBlock block = reader.next(); String logBlockInstantTime = block.getLogBlockHeader().get(INSTANT_TIME); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java index 64e1ac881e226..e73928134f081 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java @@ -25,7 +25,6 @@ import org.apache.hudi.common.schema.HoodieSchemaCache; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; -import org.apache.hudi.exception.HoodieAvroSchemaException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.io.SeekableDataInputStream; import org.apache.hudi.storage.HoodieStorage; @@ -40,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -74,9 +72,6 @@ public abstract class HoodieDataBlock extends HoodieLogBlock { protected HoodieSchema readerSchema; - // Map of string schema to parsed schema. - private static final ConcurrentHashMap SCHEMA_MAP = new ConcurrentHashMap<>(); - /** * NOTE: This ctor is used on the write-path (ie when records ought to be written into the log) */ @@ -363,21 +358,6 @@ protected Option getRecordKey(HoodieRecord record) { return Option.ofNullable(record.getRecordKey(readerSchema, keyFieldName)); } - protected HoodieSchema getSchemaFromHeader() { - String schemaStr = getLogBlockHeader().get(HeaderMetadataType.SCHEMA); - SCHEMA_MAP.computeIfAbsent(schemaStr, - (schemaString) -> { - try { - return HoodieSchema.parse(schemaStr); - } catch (HoodieAvroSchemaException e) { - // Archived commits from earlier hudi versions fail the schema check - // So we retry in this one specific instance with validation disabled - return HoodieSchema.parse(schemaStr, false); - } - }); - return SCHEMA_MAP.get(schemaStr); - } - /** * Converts the given list to closable iterator. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java index cb2923f1f2fa1..9c57ee1f65da0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java @@ -21,7 +21,7 @@ import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.model.HoodieDeleteRecord; import org.apache.hudi.avro.model.HoodieDeleteRecordList; -import org.apache.hudi.common.engine.RecordContext; +import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.fs.SizeAwareDataInputStream; import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.model.HoodieKey; @@ -137,9 +137,9 @@ public DeleteRecord[] getRecordsToDelete() { /** * Returns delete records in the same representation used by the file-group record buffer. */ - public List> getRecordsToDelete(RecordContext recordContext) { + public List> getRecordsToDelete(HoodieReaderContext readerContext) { return Arrays.stream(getRecordsToDelete()) - .map(deleteRecord -> BufferedRecords.fromDeleteRecord(deleteRecord, recordContext)) + .map(deleteRecord -> BufferedRecords.fromDeleteRecord(deleteRecord, readerContext.getRecordContext())) .collect(Collectors.toList()); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java index beb5f721a7121..3e63daa985229 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java @@ -19,10 +19,12 @@ package org.apache.hudi.common.table.log.block; import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.log.LogReaderUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.TypeUtils; +import org.apache.hudi.exception.HoodieAvroSchemaException; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.io.SeekableDataInputStream; @@ -46,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -88,6 +91,9 @@ public abstract class HoodieLogBlock { @Getter(AccessLevel.NONE) protected boolean readBlockLazily; + // Map of string schema to parsed schema. + private static final ConcurrentHashMap SCHEMA_MAP = new ConcurrentHashMap<>(); + // Return the bytes representation of the data belonging to a LogBlock public ByteArrayOutputStream getContentBytes(HoodieStorage storage) throws IOException { throw new HoodieException("No implementation was provided"); @@ -103,6 +109,21 @@ public boolean isDataOrDeleteBlock() { return getBlockType().isDataOrDeleteBlock(); } + protected HoodieSchema getSchemaFromHeader() { + String schemaStr = getLogBlockHeader().get(HeaderMetadataType.SCHEMA); + SCHEMA_MAP.computeIfAbsent(schemaStr, + schemaString -> { + try { + return HoodieSchema.parse(schemaStr); + } catch (HoodieAvroSchemaException e) { + // Archived commits from earlier hudi versions fail the schema check + // So we retry in this one specific instance with validation disabled + return HoodieSchema.parse(schemaStr, false); + } + }); + return SCHEMA_MAP.get(schemaStr); + } + public long getLogBlockLength() { throw new HoodieException("No implementation was provided"); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDataBlock.java similarity index 87% rename from hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeDataBlock.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDataBlock.java index e2ac8f1bb0cab..673707e3f920a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDataBlock.java @@ -34,7 +34,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.HashSet; import java.util.List; import java.util.Map; @@ -43,18 +42,18 @@ /** * Data block backed by a native log file. */ -public class HoodieNativeDataBlock extends HoodieDataBlock { +public class HoodieNativeLogDataBlock extends HoodieDataBlock { private final HoodieStorage storage; private final HoodieLogFile logFile; private final HoodieFileFormat fileFormat; - public HoodieNativeDataBlock(HoodieStorage storage, - HoodieLogFile logFile, - HoodieFileFormat fileFormat, - Option readerSchema, - Map header, - Map footer) { + public HoodieNativeLogDataBlock(HoodieStorage storage, + HoodieLogFile logFile, + HoodieFileFormat fileFormat, + Option readerSchema, + Map header, + Map footer) { super(Option.empty(), null, true, getContentLocation(storage, logFile), readerSchema, header, footer, HoodieRecord.RECORD_KEY_METADATA_FIELD, false); this.storage = storage; @@ -95,11 +94,7 @@ protected ClosableIterator readRecordsFromBlockPayload(HoodieReaderContex @Override protected ClosableIterator lookupEngineRecords(HoodieReaderContext readerContext, List keys, boolean fullKey) throws IOException { - return FilteringEngineRecordIterator.getInstance( - readRecordsFromBlockPayload(readerContext), - new HashSet<>(keys), - fullKey, - record -> Option.ofNullable(readerContext.getRecordContext().getRecordKey(record, readerSchema))); + return readerContext.lookupRecords(logFile.getPath(), fileFormat, readerSchema, storage, keys, fullKey); } @Override diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeDeleteBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDeleteBlock.java similarity index 50% rename from hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeDeleteBlock.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDeleteBlock.java index 37064e6ce7bb9..ddaeab29760e5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeDeleteBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieNativeLogDeleteBlock.java @@ -20,77 +20,108 @@ package org.apache.hudi.common.table.log.block; import org.apache.hudi.common.engine.HoodieReaderContext; -import org.apache.hudi.common.engine.RecordContext; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.DeleteRecord; +import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemas; import org.apache.hudi.common.table.read.BufferedRecord; import org.apache.hudi.common.table.read.BufferedRecords; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.core.io.storage.HoodieFileReader; +import org.apache.hudi.core.io.storage.HoodieIOFactory; import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; + +import static org.apache.hudi.common.util.ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER; /** * Delete block backed by a native delete log file. */ -public class HoodieNativeDeleteBlock extends HoodieDeleteBlock { +public class HoodieNativeLogDeleteBlock extends HoodieDeleteBlock { private final HoodieStorage storage; private final HoodieLogFile logFile; - private final HoodieReaderContext readerContext; private final HoodieSchema deleteLogSchema; private final List orderingFieldNames; + private final String partitionPath; + private final Properties props; + private DeleteRecord[] recordsToDelete; private List> bufferedRecordsToDelete; - public HoodieNativeDeleteBlock(HoodieStorage storage, - HoodieLogFile logFile, - HoodieReaderContext readerContext, - HoodieSchema deleteLogSchema, - List orderingFieldNames, - Map header, - Map footer) { + public HoodieNativeLogDeleteBlock(HoodieStorage storage, + HoodieLogFile logFile, + List orderingFieldNames, + String partitionPath, + Properties props, + Map header, + Map footer) { super(Option.empty(), null, true, getContentLocation(storage, logFile), header, footer); this.storage = storage; this.logFile = logFile; - this.readerContext = readerContext; - this.deleteLogSchema = deleteLogSchema; + this.deleteLogSchema = HoodieSchemas.createDeleteLogSchema(getSchemaFromHeader(), orderingFieldNames); this.orderingFieldNames = orderingFieldNames; + this.partitionPath = partitionPath; + this.props = props == null ? new Properties() : props; } @Override public DeleteRecord[] getRecordsToDelete() { - throw new HoodieNotSupportedException("Native delete log files do not support the legacy DeleteRecord[] API. " - + "Use getRecordsToDelete(RecordContext) instead. Log file: " + logFile); + if (recordsToDelete == null) { + recordsToDelete = readRecordsToDelete(); + } + return recordsToDelete; } @Override @SuppressWarnings({"rawtypes", "unchecked"}) - public List> getRecordsToDelete(RecordContext recordContext) { + public List> getRecordsToDelete(HoodieReaderContext readerContext) { if (bufferedRecordsToDelete == null) { - bufferedRecordsToDelete = (List) readBufferedRecordsToDelete(recordContext); + bufferedRecordsToDelete = (List) readBufferedRecordsToDelete(readerContext); } return (List) bufferedRecordsToDelete; } - @SuppressWarnings("unchecked") - private List> readBufferedRecordsToDelete(RecordContext recordContext) { - HoodieReaderContext typedReaderContext = (HoodieReaderContext) readerContext; + @SuppressWarnings({"rawtypes", "unchecked"}) + private DeleteRecord[] readRecordsToDelete() { + List deleteRecords = new ArrayList<>(); + String[] orderingFields = orderingFieldNames.toArray(new String[0]); + StoragePath path = logFile.getPath(); + HoodieFileFormat fileFormat = HoodieFileFormat.fromFileExtension("." + logFile.getSuffix()); + try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(storage) + .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO) + .getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, path, fileFormat, Option.empty()); + ClosableIterator recordIterator = fileReader.getRecordIterator(deleteLogSchema, deleteLogSchema)) { + while (recordIterator.hasNext()) { + HoodieRecord record = recordIterator.next(); + String recordKey = record.getRecordKey(deleteLogSchema, HoodieRecord.RECORD_KEY_METADATA_FIELD); + Comparable orderingValue = record.getOrderingValue(deleteLogSchema, props, orderingFields); + deleteRecords.add(DeleteRecord.create(recordKey, partitionPath, orderingValue)); + } + } catch (IOException e) { + throw new HoodieIOException("Failed to read native delete log file " + logFile, e); + } + return deleteRecords.toArray(new DeleteRecord[0]); + } + + private List> readBufferedRecordsToDelete(HoodieReaderContext readerContext) { List> deleteRecords = new ArrayList<>(); - try (ClosableIterator recordIterator = typedReaderContext.getFileRecordIterator( + try (ClosableIterator recordIterator = readerContext.getFileRecordIterator( logFile.getPath(), 0, FSUtils.getFileSize(storage, logFile), deleteLogSchema, deleteLogSchema, storage)) { while (recordIterator.hasNext()) { T record = recordIterator.next(); - Object recordKey = recordContext.getValue(record, deleteLogSchema, HoodieRecord.RECORD_KEY_METADATA_FIELD); - Comparable orderingValue = recordContext.getOrderingValue(record, deleteLogSchema, orderingFieldNames); + Object recordKey = readerContext.getRecordContext().getValue(record, deleteLogSchema, HoodieRecord.RECORD_KEY_METADATA_FIELD); + Comparable orderingValue = readerContext.getRecordContext().getOrderingValue(record, deleteLogSchema, orderingFieldNames); deleteRecords.add(BufferedRecords.createDelete(recordKey.toString(), orderingValue)); } } catch (IOException e) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java index f1a51b3a96498..b6c103f4949cf 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/FileGroupRecordBuffer.java @@ -301,7 +301,8 @@ private static class LogRecordIterator implements ClosableIterator fileGroupRecordBuffer) { this.fileGroupRecordBuffer = fileGroupRecordBuffer; - this.logRecordIterator = fileGroupRecordBuffer.records.iterator(); + fileGroupRecordBuffer.initializeLogRecordIterator(); + this.logRecordIterator = fileGroupRecordBuffer.logRecordIterator; } @Override diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/KeyBasedFileGroupRecordBuffer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/KeyBasedFileGroupRecordBuffer.java index afbdfd0eed5af..f40e5206e946e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/KeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/KeyBasedFileGroupRecordBuffer.java @@ -108,7 +108,7 @@ public void processNextDataRecord(BufferedRecord record, Serializable recordK @Override public void processDeleteBlock(HoodieDeleteBlock deleteBlock) throws IOException { - for (BufferedRecord record : deleteBlock.getRecordsToDelete(readerContext.getRecordContext())) { + for (BufferedRecord record : deleteBlock.getRecordsToDelete(readerContext)) { processNextDataRecord(record, record.getRecordKey()); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/PositionBasedFileGroupRecordBuffer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/PositionBasedFileGroupRecordBuffer.java index 62a851a4b9d31..2ca86d6ad7a52 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/PositionBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/PositionBasedFileGroupRecordBuffer.java @@ -38,6 +38,7 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieKeyException; import lombok.extern.slf4j.Slf4j; @@ -146,6 +147,13 @@ public void processDataBlock(HoodieDataBlock dataBlock, Option keySpecO BufferedRecord bufferedRecord = BufferedRecords.fromEngineRecord(projectedNextRecord, schema, readerContext.getRecordContext(), orderingFieldNames, isDelete); processNextDataRecord(bufferedRecord, recordPosition); } + if (recordIndex != recordPositions.size()) { + throw new HoodieException(String.format( + "Data block yields %d records but its header has %d record positions. Position based " + + "merging pairs records with positions by index, so records must not be dropped from " + + "log file reads (e.g. by filter pushdown into the scan).", + recordIndex, recordPositions.size())); + } } } @@ -184,10 +192,18 @@ public void processDeleteBlock(HoodieDeleteBlock deleteBlock) throws IOException return; } + List> deleteRecords = deleteBlock.getRecordsToDelete(readerContext); + if (deleteRecords.size() != recordPositions.size()) { + throw new HoodieException(String.format( + "Delete block yields %d records but its header has %d record positions. Position based " + + "merging pairs records with positions by index, so records must not be dropped from " + + "log file reads (e.g. by filter pushdown into the scan).", + deleteRecords.size(), recordPositions.size())); + } + switch (recordMergeMode) { case COMMIT_TIME_ORDERING: int commitTimeBasedRecordIndex = 0; - List> deleteRecords = deleteBlock.getRecordsToDelete(readerContext.getRecordContext()); for (Long recordPosition : recordPositions) { // IMPORTANT: // use #put for log files with regular order(see HoodieLogFile.LOG_FILE_COMPARATOR); @@ -205,7 +221,7 @@ public void processDeleteBlock(HoodieDeleteBlock deleteBlock) throws IOException case CUSTOM: default: int recordIndex = 0; - for (BufferedRecord record : deleteBlock.getRecordsToDelete(readerContext.getRecordContext())) { + for (BufferedRecord record : deleteRecords) { long recordPosition = recordPositions.get(recordIndex++); processNextDataRecord(record, recordPosition); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/ReusableFileGroupRecordBufferLoader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/ReusableFileGroupRecordBufferLoader.java index bd3b3ec55bb36..3e1def1a16e2a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/ReusableFileGroupRecordBufferLoader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/ReusableFileGroupRecordBufferLoader.java @@ -63,7 +63,8 @@ public synchronized Pair, List> getRecord if (cachedResults == null) { // Create an initial buffer to process the log files KeyBasedFileGroupRecordBuffer initialBuffer = new KeyBasedFileGroupRecordBuffer<>( - readerContext, hoodieTableMetaClient, readerContext.getMergeMode(), partialUpdateModeOpt, props, orderingFieldNames, updateProcessor); + readerContextWithoutFilters, hoodieTableMetaClient, readerContextWithoutFilters.getMergeMode(), + partialUpdateModeOpt, props, orderingFieldNames, updateProcessor); List validInstants = scanLogFiles(readerContextWithoutFilters, storage, inputSplit, hoodieTableMetaClient, props, readerParameters, readStats, initialBuffer); cachedResults = Pair.of(initialBuffer, validInstants); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineLoaderV1.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineLoaderV1.java index 1ac715b379fd3..5eae55e929f3c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineLoaderV1.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v1/ArchivedTimelineLoaderV1.java @@ -122,8 +122,8 @@ public void loadInstants(HoodieTableMetaClient metaClient, continue; } // Read the archived file - try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(metaClient.getStorage(), - new HoodieLogFile(fs.getPath()), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) { + try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(metaClient, new HoodieLogFile(fs.getPath()), + HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) { int instantsInPreviousFile = instantsInRange.size(); // Read the avro blocks while (reader.hasNext() && (!hasLimit || loadedCount.get() < limit.get())) { diff --git a/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieAvroFileWriter.java b/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieAvroFileWriter.java index dd23bc5b48f4b..e401f99ba85ce 100644 --- a/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieAvroFileWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieAvroFileWriter.java @@ -29,7 +29,7 @@ import java.io.IOException; import java.util.Properties; -public interface HoodieAvroFileWriter extends HoodieFileWriter { +public interface HoodieAvroFileWriter extends HoodieFileWriter { boolean canWrite(); @@ -51,6 +51,11 @@ default void write(String recordKey, HoodieRecord record, HoodieSchema schema, P writeAvro(recordKey, avroPayload); } + @Override + default void writeRow(String recordKey, IndexedRecord record) throws IOException { + writeAvro(recordKey, record); + } + default void prepRecordWithMetadata(HoodieKey key, IndexedRecord avroRecord, String instantTime, Integer partitionId, long recordIndex, String fileName) { String seqId = HoodieRecord.generateSequenceId(instantTime, partitionId, recordIndex); HoodieAvroUtils.addHoodieKeyToRecord((GenericRecord) avroRecord, key.getRecordKey(), key.getPartitionPath(), fileName); diff --git a/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieFileWriter.java b/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieFileWriter.java index fe2a3b1803d3e..4534a8ee308b4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieFileWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieFileWriter.java @@ -27,13 +27,15 @@ import java.util.Map; import java.util.Properties; -public interface HoodieFileWriter extends AutoCloseable { +public interface HoodieFileWriter extends AutoCloseable { boolean canWrite(); void writeWithMetadata(HoodieKey key, HoodieRecord record, HoodieSchema schema, Properties props) throws IOException; void write(String recordKey, HoodieRecord record, HoodieSchema schema, Properties props) throws IOException; + void writeRow(String recordKey, T record) throws IOException; + void close() throws IOException; default void addFooterMetadata(Map footerMetadata) { diff --git a/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieNativeAvroHFileReader.java b/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieNativeAvroHFileReader.java index 5c2ed5b994aa5..5ef06e1823626 100644 --- a/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieNativeAvroHFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieNativeAvroHFileReader.java @@ -57,7 +57,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -101,13 +100,16 @@ public ClosableIterator getIndexedRecordIterator(HoodieSchema rea HoodieSchema requestedSchema, Map renamedColumns) throws IOException { - if (!Objects.equals(readerSchema, requestedSchema) || !renamedColumns.isEmpty()) { + // GenericDatumReader natively supports Avro projection from the stored writer schema + // to the requested read schema. Column rename rewriting still needs file-level schema + // handling that this reader does not implement. + if (!renamedColumns.isEmpty()) { throw new UnsupportedOperationException( - "Schema projections are not supported in HFile reader"); + "Column renames are not supported in HFile reader"); } HFileReader reader = readerFactory.createHFileReader(); - return new RecordIterator(reader, getSchema(), readerSchema); + return new RecordIterator(reader, getSchema(), requestedSchema); } @Override diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java index 40a39542c7a71..66cdef3f4ce56 100644 --- a/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java @@ -51,6 +51,21 @@ public static boolean validateErrorMsg(@Nonnull Throwable t, String errorMsg) { return false; } + /** + * Returns true if any nested exception is an instance of the provided exception class. + */ + public static boolean isCausedBy(@Nonnull Throwable t, @Nonnull Class clazz) { + Throwable cause = t; + while (cause != null) { + if (clazz.isInstance(cause)) { + return true; + } + cause = cause.getCause(); + } + + return false; + } + /** * Throws the provided exception as-is when it is an {@link IOException} or * {@link RuntimeException}, otherwise wraps it in an {@link IOException}. diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index 0fb325bd30537..ec83e4c080e86 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -877,8 +877,10 @@ public static HoodieData convertMetadataToRecordIndexRecords(H EngineType engineType) { List allWriteStats = commitMetadata.getPartitionToWriteStats().values().stream() .flatMap(Collection::stream).collect(Collectors.toList()); - // Return early if there are no write stats, or if the operation is a compaction. - if (allWriteStats.isEmpty() || commitMetadata.getOperationType() == WriteOperationType.COMPACT) { + // Return early if there are no write stats, or if the operation is compaction or log compaction. + if (allWriteStats.isEmpty() + || commitMetadata.getOperationType() == WriteOperationType.COMPACT + || commitMetadata.getOperationType() == WriteOperationType.LOG_COMPACT) { return engineContext.emptyHoodieData(); } // RLI cannot support logs having inserts with current offering. So, lets validate that. @@ -893,7 +895,6 @@ public static HoodieData convertMetadataToRecordIndexRecords(H Map> writeStatsByFileId = allWriteStats.stream().collect(Collectors.groupingBy(HoodieWriteStat::getFileId)); int parallelism = Math.max(Math.min(writeStatsByFileId.size(), metadataConfig.getRecordIndexMaxParallelism()), 1); String basePath = dataTableMetaClient.getBasePath().toString(); - HoodieFileFormat baseFileFormat = dataTableMetaClient.getTableConfig().getBaseFileFormat(); StorageConfiguration storageConfiguration = dataTableMetaClient.getStorageConf(); Option writerSchemaOpt = tryResolveSchemaForTable(dataTableMetaClient); Option finalWriterSchemaOpt = writerSchemaOpt; @@ -904,10 +905,10 @@ public static HoodieData convertMetadataToRecordIndexRecords(H List writeStats = writeStatsByFileIdEntry.getValue(); // Partition the write stats into base file and log file write stats List baseFileWriteStats = writeStats.stream() - .filter(writeStat -> writeStat.getPath().endsWith(baseFileFormat.getFileExtension())) + .filter(writeStat -> FSUtils.isBaseFile(new StoragePath(writeStat.getPath()))) .collect(Collectors.toList()); List logFileWriteStats = writeStats.stream() - .filter(writeStat -> FSUtils.isLogFile(new StoragePath(writeStats.get(0).getPath()))) + .filter(writeStat -> FSUtils.isLogFile(new StoragePath(writeStat.getPath()))) .collect(Collectors.toList()); // Ensure that only one of base file or log file write stats exists checkState(baseFileWriteStats.isEmpty() || logFileWriteStats.isEmpty(), @@ -1771,6 +1772,11 @@ private static List> readColumnRangeMetada : partitionPath + "/" + fileName; try { StoragePath fullFilePath = new StoragePath(datasetMetaClient.getBasePath(), partitionPathFileName); + if (FSUtils.isNativeDeleteLogFile(fileName)) { + return columnsToIndex.stream() + .map(column -> HoodieColumnRangeMetadata.createEmpty(fileName, column, indexVersion)) + .collect(Collectors.toList()); + } if (partitionPathFileName.endsWith(HoodieFileFormat.PARQUET.getFileExtension())) { return HoodieIOFactory.getIOFactory(datasetMetaClient.getStorage()) .getFileFormatUtils(HoodieFileFormat.PARQUET) diff --git a/hudi-common/src/test/java/org/apache/hudi/common/engine/TestHoodieReaderContext.java b/hudi-common/src/test/java/org/apache/hudi/common/engine/TestHoodieReaderContext.java new file mode 100644 index 0000000000000..b74b479790dbe --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/engine/TestHoodieReaderContext.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.engine; + +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.expression.Predicate; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.log.InstantRange; +import org.apache.hudi.common.table.read.BufferedRecord; +import org.apache.hudi.common.util.DefaultJavaTypeConverter; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; + +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.UnaryOperator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TestHoodieReaderContext { + + private static final HoodieSchema READER_SCHEMA = HoodieSchema.createRecord("TestRecord", null, null, new ArrayList<>()); + + @Test + void defaultLookupRecordsFiltersByFullKey() throws IOException { + TestReaderContext readerContext = new TestReaderContext(Arrays.asList("id1", "id2", "id20")); + + List actual = lookup(readerContext, Arrays.asList("id2", "missing"), true); + + assertEquals(Arrays.asList("id2"), actual); + } + + @Test + void defaultLookupRecordsFiltersByKeyPrefix() throws IOException { + TestReaderContext readerContext = new TestReaderContext(Arrays.asList("id1", "id2", "id20", "other")); + + List actual = lookup(readerContext, Arrays.asList("id2"), false); + + assertEquals(Arrays.asList("id2", "id20"), actual); + } + + private static List lookup(TestReaderContext readerContext, List keys, boolean fullKey) throws IOException { + StoragePath filePath = new StoragePath("/tmp/test.parquet"); + HoodieStorage storage = mock(HoodieStorage.class); + when(storage.getPathInfo(filePath)).thenReturn(new StoragePathInfo(filePath, 100, false, (short) 1, 100, 0)); + + List result = new ArrayList<>(); + try (ClosableIterator iterator = readerContext.lookupRecords( + filePath, HoodieFileFormat.PARQUET, READER_SCHEMA, storage, keys, fullKey)) { + iterator.forEachRemaining(result::add); + } + return result; + } + + private static class TestReaderContext extends HoodieReaderContext { + private final List records; + + private TestReaderContext(List records) { + super(mock(StorageConfiguration.class), tableConfig(), Option.empty(), Option.empty(), new TestRecordContext()); + this.records = records; + } + + @Override + public ClosableIterator getFileRecordIterator( + StoragePath filePath, + long start, + long length, + HoodieSchema dataSchema, + HoodieSchema requiredSchema, + HoodieStorage storage) { + return ClosableIterator.wrap(records.iterator()); + } + + @Override + protected Option getRecordMerger(RecordMergeMode mergeMode, String mergeStrategyId, String mergeImplClasses) { + return Option.empty(); + } + + @Override + public ClosableIterator mergeBootstrapReaders( + ClosableIterator skeletonFileIterator, + HoodieSchema skeletonRequiredSchema, + ClosableIterator dataFileIterator, + HoodieSchema dataRequiredSchema, + List> requiredPartitionFieldAndValues) { + return ClosableIterator.wrap(new ArrayList().iterator()); + } + + private static HoodieTableConfig tableConfig() { + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getBaseFileFormat()).thenReturn(HoodieFileFormat.PARQUET); + return tableConfig; + } + } + + private static class TestRecordContext extends RecordContext { + private TestRecordContext() { + super(new DefaultJavaTypeConverter()); + } + + @Override + public String getRecordKey(String record, HoodieSchema schema) { + return record; + } + + @Override + public HoodieRecord constructHoodieRecord(BufferedRecord bufferedRecord, String partitionPath) { + return null; + } + + @Override + public String mergeWithEngineRecord(HoodieSchema schema, Map updateValues, BufferedRecord baseRecord) { + return null; + } + + @Override + public String constructEngineRecord(HoodieSchema recordSchema, Object[] fieldValues) { + return null; + } + + @Override + public HoodieRecord.HoodieRecordType getEngineRecordType() { + return HoodieRecord.HoodieRecordType.AVRO; + } + + @Override + public Object getValue(String record, HoodieSchema schema, String fieldName) { + return null; + } + + @Override + public String getMetaFieldValue(String record, int pos) { + return null; + } + + @Override + public String convertAvroRecord(IndexedRecord avroRecord) { + return null; + } + + @Override + public GenericRecord convertToAvroRecord(String record, HoodieSchema schema) { + return null; + } + + @Override + public String getDeleteRow(String recordKey) { + return null; + } + + @Override + public String seal(String record) { + return record; + } + + @Override + public String toBinaryRow(HoodieSchema schema, String record) { + return record; + } + + @Override + public UnaryOperator projectRecord(HoodieSchema from, HoodieSchema to, Map renamedColumns) { + return UnaryOperator.identity(); + } + } +} diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/SchemaHandlerTestBase.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/SchemaHandlerTestBase.java index c59923cfa232e..123760e689fd9 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/SchemaHandlerTestBase.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/SchemaHandlerTestBase.java @@ -358,6 +358,11 @@ public String constructEngineRecord(HoodieSchema recordSchema, Object[] fieldVal return ""; } + @Override + public HoodieRecord.HoodieRecordType getEngineRecordType() { + return HoodieRecord.HoodieRecordType.AVRO; + } + @Override public String seal(String record) { return ""; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/BaseTestFileGroupRecordBuffer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/BaseTestFileGroupRecordBuffer.java index d39914a9b9813..8d75a72ee92d9 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/BaseTestFileGroupRecordBuffer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/BaseTestFileGroupRecordBuffer.java @@ -93,9 +93,9 @@ protected static GenericRecord createTestRecord(String recordKey, int counter, l protected static void mockDeleteRecords(HoodieDeleteBlock deleteBlock, DeleteRecord... deleteRecords) { when(deleteBlock.getRecordsToDelete()).thenReturn(deleteRecords); when(deleteBlock.getRecordsToDelete(any())).thenAnswer(invocation -> { - RecordContext recordContext = invocation.getArgument(0); + HoodieReaderContext readerContext = invocation.getArgument(0); return Arrays.stream(deleteRecords) - .map(deleteRecord -> BufferedRecords.fromDeleteRecord(deleteRecord, recordContext)) + .map(deleteRecord -> BufferedRecords.fromDeleteRecord(deleteRecord, readerContext.getRecordContext())) .collect(Collectors.toList()); }); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileGroupRecordIterator.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileGroupRecordIterator.java index 01a90cfd08e9c..0810f1e12ed08 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileGroupRecordIterator.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/lsm/TestLsmFileGroupRecordIterator.java @@ -59,7 +59,10 @@ void testDeleteLogSchemaUsesRecordKeyAndOrderingFields() { .map(HoodieSchemaField::name) .collect(Collectors.toList())); assertEquals(HoodieSchemaType.STRING, deleteLogSchema.getField(HoodieRecord.RECORD_KEY_METADATA_FIELD).get().schema().getType()); - assertEquals(HoodieSchemaType.LONG, deleteLogSchema.getField("ts").get().schema().getType()); + HoodieSchemaField orderingField = deleteLogSchema.getField("ts").get(); + assertTrue(orderingField.schema().isNullable()); + assertEquals(HoodieSchemaType.LONG, orderingField.getNonNullSchema().getType()); + assertEquals(HoodieSchema.NULL_VALUE, orderingField.defaultVal().get()); } @Test diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java index 99ea61ffb5937..f736b0506d801 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java @@ -20,6 +20,7 @@ import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieAvroIndexedRecord; import org.apache.hudi.common.model.HoodieFileFormat; @@ -153,6 +154,24 @@ public static HoodieStorage getStorage(StoragePath path) { return HoodieStorageUtils.getStorage(path, getDefaultStorageConf()); } + /** + * Lists all native (v2) log files (e.g. {@code *.log.parquet} / {@code *.deletes.parquet}) under + * the given base path, scanning recursively. + */ + public static List listNativeLogFiles(HoodieStorage storage, String basePath) throws IOException { + return storage.listFiles(new StoragePath(basePath)).stream() + .map(StoragePathInfo::getPath) + .filter(path -> FSUtils.matchNativeLogFile(path.getName()).isPresent()) + .collect(Collectors.toList()); + } + + /** + * Whether the table at the given base path contains any native (v2) log files. + */ + public static boolean hasNativeLogFiles(HoodieStorage storage, String basePath) throws IOException { + return !listNativeLogFiles(storage, basePath).isEmpty(); + } + public static HoodieTableMetaClient init(String basePath) throws IOException { return init(basePath, HoodieTableType.COPY_ON_WRITE); } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java index ddc3911a6014f..7d163ccc50979 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieOperation; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.schema.HoodieSchema; @@ -38,6 +39,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.common.util.collection.CloseableMappingIterator; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.core.io.storage.HoodieIOFactory; import org.apache.hudi.exception.HoodieException; @@ -53,6 +55,7 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.utils.JoinedRowData; import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.types.RowKind; import java.io.IOException; import java.util.Arrays; @@ -98,24 +101,39 @@ public ClosableIterator getFileRecordIterator( HoodieSchema dataSchema, HoodieSchema requiredSchema, HoodieStorage storage) throws IOException { - boolean isLogFile = FSUtils.isLogFile(filePath); // disable schema evolution in fileReader if it's log file, since schema evolution for log file is handled in `FileGroupRecordBuffer` - InternalSchemaManager schemaManager = isLogFile ? InternalSchemaManager.DISABLED : internalSchemaManager.get(); + InternalSchemaManager schemaManager = FSUtils.isLogFile(filePath) ? InternalSchemaManager.DISABLED : internalSchemaManager.get(); // Log files only reach this method for parquet data blocks; base files are resolved by their extension. // Format-specific handling lives in the readers themselves, so this method stays format-agnostic. - HoodieFileFormat format = isLogFile ? HoodieFileFormat.PARQUET : HoodieFileFormat.fromFileExtension(filePath.getFileExtension()); + boolean isInlineLogFile = FSUtils.isLogFile(filePath) && FSUtils.matchNativeLogFile(filePath.getName()).isEmpty(); + HoodieFileFormat format = isInlineLogFile ? HoodieFileFormat.PARQUET : HoodieFileFormat.fromFileExtension(filePath.getFileExtension()); HoodieRowDataFileReader reader = (HoodieRowDataFileReader) HoodieIOFactory.getIOFactory(storage) .getReaderFactory(HoodieRecord.HoodieRecordType.FLINK) .getFileReader(tableConfig, filePath, format, Option.empty()); try { - return reader.getRowDataIterator(dataSchema, requiredSchema, schemaManager, getSafePredicates(requiredSchema)); + ClosableIterator rowDataIterator = + reader.getRowDataIterator(dataSchema, requiredSchema, schemaManager, getSafePredicates(requiredSchema)); + return resolveRowKind(rowDataIterator, requiredSchema); } catch (Throwable e) { reader.close(); throw new HoodieException("Failed to get record iterator for: " + filePath, e); } } + private ClosableIterator resolveRowKind(ClosableIterator rowDataIterator, HoodieSchema requiredSchema) { + Option operationField = requiredSchema.getField(HoodieRecord.OPERATION_METADATA_FIELD); + if (!operationField.isPresent()) { + return rowDataIterator; + } + int operationPos = operationField.get().pos(); + return new CloseableMappingIterator<>(rowDataIterator, rowData -> { + HoodieOperation operation = HoodieOperation.fromName(rowData.getString(operationPos).toString()); + rowData.setRowKind(RowKind.fromByteValue(operation.getValue())); + return rowData; + }); + } + @Override public Option getRecordMerger(RecordMergeMode mergeMode, String mergeStrategyId, String mergeImplClasses) { switch (mergeMode) { diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java index 7b2ea97f9468d..353ae07489710 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java @@ -360,7 +360,7 @@ public static class Builder { protected Configuration conf; protected MergeOnReadTableState tableState; protected List fieldTypes; - protected List predicates; + protected List predicates = Collections.emptyList(); protected long limit = -1; protected boolean emitDelete = false; protected InternalSchemaManager internalSchemaManager = InternalSchemaManager.DISABLED; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java index 69087122a2e81..12357e42d4a16 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java @@ -639,10 +639,6 @@ protected Map getUpsertWithDeleteExpected() { return expected; } - protected Map getExpectedBeforeCheckpointComplete() { - return EXPECTED2; - } - @Test public void testIndexStateBootstrap() throws Exception { // open the function and ingest data @@ -678,7 +674,7 @@ protected void validateIndexLoaded() throws Exception { .checkpoint(1) .assertBootstrapped() .assertNextEvent() - .checkWrittenData(getExpectedBeforeCheckpointComplete()) + .checkWrittenData(EXPECTED1) .checkpointComplete(1) .checkWrittenData(EXPECTED2) .end(); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java index 5e4deee54c966..c3fa251119ae7 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java @@ -379,11 +379,6 @@ public void testInsertDuplicateRecordsWithCDCMode() throws Exception { assertEquals(firstInstant.requestedTime(), secondWriteStats.get(0).getPrevCommit()); } - @Override - protected Map getExpectedBeforeCheckpointComplete() { - return EXPECTED1; - } - protected Map getMiniBatchExpected() { Map expected = new HashMap<>(); // MOR mode merges the messages with the same key. diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java index 3f24c2763faf1..36e0f8d0b80fb 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java @@ -107,11 +107,6 @@ public void testInsertAsyncClustering() { // insert async clustering is only valid for cow table. } - @Override - protected Map getExpectedBeforeCheckpointComplete() { - return EXPECTED1; - } - protected Map getMiniBatchExpected() { Map expected = new HashMap<>(); // MOR mode merges the messages with the same key. diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bulk/TestBulkInsertWriteHelper.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bulk/TestBulkInsertWriteHelper.java index 38f75ca7ca508..773e0eefd3e54 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bulk/TestBulkInsertWriteHelper.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bulk/TestBulkInsertWriteHelper.java @@ -60,7 +60,7 @@ public class TestBulkInsertWriteHelper { @BeforeEach public void before() throws IOException { - conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); + conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath(), TestConfigurations.ROW_DATA_TYPE); StreamerUtil.initTableIfNotExists(conf); } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestFlinkConsistentHashingClustering.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestFlinkConsistentHashingClustering.java index ee9ef6ae065b0..b59e47d1f5bcd 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestFlinkConsistentHashingClustering.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestFlinkConsistentHashingClustering.java @@ -127,7 +127,7 @@ private void prepareData(TableEnvironment tableEnv) throws Exception { TimeUnit.SECONDS.sleep(3); // Validate the insertion - TestData.checkWrittenData(tempFile, EXPECTED_AFTER_INITIAL_INSERT, 0); + TestData.checkWrittenDataMOR(tempFile, EXPECTED_AFTER_INITIAL_INSERT, 0); } private TableEnvironment setupTableEnv() { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java index 3ae8ea520f2cf..a4e85e17dfd78 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java @@ -195,8 +195,9 @@ public void testHoodieFlinkCompactor(boolean enableChangelog, String logBlockFor } } + // todo: enable test for downgrade after this issue solved: https://github.com/apache/hudi/issues/19090 @ParameterizedTest - @ValueSource(booleans = {true, false}) + @ValueSource(booleans = {true}) void testHoodieFlinkCompactorWithUpgradeAndDowngrade(boolean upgrade) throws Exception { // Create hoodie table and insert into data. EnvironmentSettings settings = EnvironmentSettings.newInstance().inBatchMode().build(); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/TestWriteBase.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/TestWriteBase.java index 36c2e1ba1deaa..b051cad77a6e3 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/TestWriteBase.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/TestWriteBase.java @@ -20,6 +20,7 @@ import org.apache.hudi.client.HoodieFlinkWriteClient; import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; @@ -586,7 +587,7 @@ private boolean fileExists(boolean ignoreLogs) { } } else { if (ignoreLogs) { - if (!file.getName().startsWith(".")) { + if (FSUtils.isBaseFile(file.getName())) { return true; } } else { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java index 21910df52499a..9cf41d1e47fd3 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java @@ -2339,8 +2339,13 @@ void testIncrementalReadArchivedCommits(HoodieTableType tableType) throws Except 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)); } + // Only test COPY_ON_WRITE here: the file group reader reads records using the table (write) schema + // persisted in the timeline, not the wider query schema defined by the Flink catalog DDL. So columns + // that only exist in the DDL (e.g. the newly added `salary`) are not recognized and reading fails with + // "One or more specified columns does not exist in the hudi table". MERGE_ON_READ goes through the file + // group reader path and therefore does not support reading with a wider schema in this scenario. @ParameterizedTest - @EnumSource(value = HoodieTableType.class) + @EnumSource(value = HoodieTableType.class, names = {"COPY_ON_WRITE"}) void testReadWithWiderSchema(HoodieTableType tableType) throws Exception { TableEnvironment tableEnv = batchTableEnv; Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath()); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java index 8e23abb9563f0..b83b023305578 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java @@ -42,7 +42,7 @@ import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode; import org.apache.hudi.common.table.log.HoodieLogFormat; -import org.apache.hudi.common.table.log.block.HoodieHFileDataBlock; +import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -209,7 +209,8 @@ void testRecordIndexLogBlocksContainBloomFilter() throws Exception { TestData.writeData(TestData.DATA_SET_INSERT, conf); - HoodieStorage storage = StreamerUtil.createMetaClient(conf).getStorage(); + HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(conf); + HoodieStorage storage = metaClient.getStorage(); StoragePath recordIndexPath = new StoragePath( HoodieTableMetadata.getMetadataTableBasePath(conf.get(FlinkOptions.PATH)), MetadataPartitionType.RECORD_INDEX.getPartitionPath()); @@ -220,12 +221,12 @@ void testRecordIndexLogBlocksContainBloomFilter() throws Exception { assertFalse(logFiles.isEmpty(), "The MDT record index partition should contain log files"); int hfileDataBlockCount = 0; for (StoragePathInfo logFile : logFiles) { - HoodieSchema schema = TableSchemaResolver.readSchemaFromLogFile(storage, logFile.getPath()); + HoodieSchema schema = TableSchemaResolver.readSchemaFromLogFile(metaClient, logFile.getPath()); try (HoodieLogFormat.Reader reader = - HoodieLogFormat.newReader(storage, new HoodieLogFile(logFile), schema)) { + HoodieLogFormat.newReader(metaClient, new HoodieLogFile(logFile), schema)) { while (reader.hasNext()) { HoodieLogBlock logBlock = reader.next(); - if (logBlock instanceof HoodieHFileDataBlock) { + if (!(logBlock instanceof HoodieDeleteBlock)) { assertBloomFilterContainsWrittenKey(storage, logBlock); hfileDataBlockCount++; } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java index e227e8bd71d40..30abdab3f2830 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java @@ -21,7 +21,6 @@ import org.apache.hudi.client.common.HoodieFlinkEngineContext; import org.apache.hudi.client.model.PartialUpdateFlinkRecordMerger; import org.apache.hudi.common.config.HoodieMemoryConfig; -import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.model.PartialUpdateAvroPayload; @@ -840,27 +839,54 @@ public static void checkWrittenData( Map expected, int partitions, Function extractor) throws IOException { - assert baseFile.isDirectory(); - FileFilter filter = file -> !file.getName().startsWith("."); - File[] partitionDirs = baseFile.listFiles(filter); + assert baseFile.isDirectory() : "Base path should be a directory"; + String basePath = baseFile.getAbsolutePath(); + File[] partitionDirs = baseFile.listFiles(file -> !file.getName().startsWith(".") && file.isDirectory()); assertNotNull(partitionDirs); - assertThat(partitionDirs.length, is(partitions)); - for (File partitionDir : partitionDirs) { - File[] dataFiles = partitionDir.listFiles(filter); - assertNotNull(dataFiles); - File latestDataFile = Arrays.stream(dataFiles) - .max(Comparator.comparing(f -> FSUtils.getCommitTime(f.getName()))) - .orElse(dataFiles[0]); - ParquetReader reader = AvroParquetReader - .builder(new Path(latestDataFile.getAbsolutePath())).build(); + assertThat("The partitions number should be: " + partitions, partitionDirs.length, is(partitions)); + + HoodieWriteConfig config = HoodieWriteConfig.newBuilder() + .withPath(baseFile.toURI().toString()) + .withMemoryConfig(getReaderMemoryConfig()) + .build(); + HoodieTableMetaClient metaClient = createMetaClient(basePath); + HoodieFlinkTable table = HoodieFlinkTable.create(config, HoodieFlinkEngineContext.DEFAULT, metaClient); + boolean hasCompletedInstant = metaClient.getActiveTimeline().filterCompletedInstants().lastInstant().isPresent(); + Map> partitionToFileSlices = new HashMap<>(); + for (Map.Entry partitionAndExpected : expected.entrySet()) { + List fileSlices = (hasCompletedInstant + ? table.getSliceView().getLatestFileSlices(partitionAndExpected.getKey()) + : table.getSliceView().getLatestFileSlicesIncludingInflight(partitionAndExpected.getKey())) + .filter(fileSlice -> fileSlice.getBaseFile().isPresent()) + .map(fileSlice -> fileSlice.withLogFiles(false)) + .collect(Collectors.toList()); + partitionToFileSlices.put(partitionAndExpected.getKey(), fileSlices); + } + + HoodieSchema schema = getTableSchema(metaClient, basePath); + RowDataQueryContexts.RowDataQueryContext queryContext = RowDataQueryContexts.fromSchema(schema); + + for (Map.Entry partitionAndExpected : expected.entrySet()) { List readBuffer = new ArrayList<>(); - GenericRecord nextRecord = reader.read(); - while (nextRecord != null) { - readBuffer.add(extractor.apply(nextRecord)); - nextRecord = reader.read(); + List fileSlices = partitionToFileSlices.get(partitionAndExpected.getKey()); + for (FileSlice fileSlice : fileSlices) { + try (ClosableIterator rowIterator = getRecordIterator(fileSlice, schema, metaClient, config)) { + while (rowIterator.hasNext()) { + RowData rowData = rowIterator.next(); + readBuffer.add(extractor.apply((GenericRecord) queryContext.getRowDataToAvroConverter().convert(schema, rowData))); + } + } } readBuffer.sort(Comparator.naturalOrder()); - assertThat(readBuffer.toString(), is(expected.get(partitionDir.getName()))); + assertThat(readBuffer.toString(), is(partitionAndExpected.getValue())); + } + } + + private static HoodieSchema getTableSchema(HoodieTableMetaClient metaClient, String basePath) throws IOException { + try { + return new TableSchemaResolver(metaClient).getTableSchema(); + } catch (Exception e) { + throw new IOException("Failed to resolve table schema under table path " + basePath, e); } } @@ -1015,26 +1041,8 @@ public static void checkWrittenDataMOR( int partitions) throws Exception { assert baseFile.isDirectory() : "Base path should be a directory"; String basePath = baseFile.getAbsolutePath(); - File hoodiePropertiesFile = new File(baseFile + "/" + METAFOLDER_NAME + "/" + HOODIE_PROPERTIES_FILE); - assert hoodiePropertiesFile.exists(); // 1. init flink table - HoodieWriteConfig config = HoodieWriteConfig.newBuilder() - .fromFile(hoodiePropertiesFile) - .withMemoryConfig( - HoodieMemoryConfig.newBuilder() - .withMaxMemoryMaxSize( - FlinkOptions.WRITE_MERGE_MAX_MEMORY.defaultValue() * 1024 * 1024L, - FlinkOptions.COMPACTION_MAX_MEMORY.defaultValue() * 1024 * 1024L) - .build()) - .withPath(basePath) - .build(); - // deal with partial update merger - if (config.getString(HoodieTableConfig.PAYLOAD_CLASS_NAME).contains(PartialUpdateAvroPayload.class.getSimpleName()) - || (config.getString(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID) != null - && config.getString(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID).equalsIgnoreCase(HoodieRecordMerger.CUSTOM_MERGE_STRATEGY_UUID))) { - config.setValue(HoodieWriteConfig.RECORD_MERGE_IMPL_CLASSES.key(), PartialUpdateFlinkRecordMerger.class.getName()); - } - + HoodieWriteConfig config = getMORWriteConfig(baseFile); HoodieTableMetaClient metaClient = createMetaClient(basePath); HoodieFlinkTable table = HoodieFlinkTable.create(config, HoodieFlinkEngineContext.DEFAULT, metaClient); HoodieSchema schema = new TableSchemaResolver(metaClient).getTableSchema(); @@ -1092,6 +1100,31 @@ private static ClosableIterator getRecordIterator( return fileGroupReader.getClosableIterator(); } + private static HoodieWriteConfig getMORWriteConfig(File baseFile) throws IOException { + String basePath = baseFile.getAbsolutePath(); + File hoodiePropertiesFile = new File(baseFile + "/" + METAFOLDER_NAME + "/" + HOODIE_PROPERTIES_FILE); + assert hoodiePropertiesFile.exists(); + HoodieWriteConfig config = HoodieWriteConfig.newBuilder() + .fromFile(hoodiePropertiesFile) + .withMemoryConfig(getReaderMemoryConfig()) + .withPath(basePath) + .build(); + if (config.getString(HoodieTableConfig.PAYLOAD_CLASS_NAME).contains(PartialUpdateAvroPayload.class.getSimpleName()) + || (config.getString(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID) != null + && config.getString(HoodieTableConfig.RECORD_MERGE_STRATEGY_ID).equalsIgnoreCase(HoodieRecordMerger.CUSTOM_MERGE_STRATEGY_UUID))) { + config.setValue(HoodieWriteConfig.RECORD_MERGE_IMPL_CLASSES.key(), PartialUpdateFlinkRecordMerger.class.getName()); + } + return config; + } + + private static HoodieMemoryConfig getReaderMemoryConfig() { + return HoodieMemoryConfig.newBuilder() + .withMaxMemoryMaxSize( + FlinkOptions.WRITE_MERGE_MAX_MEMORY.defaultValue() * 1024 * 1024L, + FlinkOptions.COMPACTION_MAX_MEMORY.defaultValue() * 1024 * 1024L) + .build(); + } + /** * Filter out the variables like file name. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/LanceUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/LanceUtils.java similarity index 77% rename from hudi-common/src/main/java/org/apache/hudi/common/util/LanceUtils.java rename to hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/LanceUtils.java index fe278bb4faa92..8a64f0b8eaf2e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/LanceUtils.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/LanceUtils.java @@ -20,6 +20,7 @@ package org.apache.hudi.common.util; import org.apache.hudi.common.config.HoodieReaderConfig; +import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -29,17 +30,23 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.core.io.storage.HoodieFileReader; import org.apache.hudi.core.io.storage.HoodieIOFactory; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.MetadataNotFoundException; +import org.apache.hudi.io.memory.HoodieArrowAllocator; import org.apache.hudi.keygen.BaseKeyGenerator; import org.apache.hudi.metadata.HoodieIndexVersion; import org.apache.hudi.stats.HoodieColumnRangeMetadata; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; +import org.apache.arrow.memory.BufferAllocator; import org.apache.avro.generic.GenericRecord; +import org.lance.file.LanceFileReader; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -87,13 +94,10 @@ public ClosableIterator getHoodieKeyIterator(HoodieStorage storage, record -> { String recordKey; if (keyGeneratorOpt.isPresent()) { - // With keyGenerator, extracts user-defined key fields by name recordKey = record.getRecordKey(keySchema, keyGeneratorOpt); } else { - // Without keyGenerator, read record key field by name recordKey = record.getRecordKey(keySchema, HoodieRecord.RECORD_KEY_METADATA_FIELD); } - // Extract partition path from record if not provided as parameter String partitionPathValue = partitionPath.orElseGet(() -> (String) record.getColumnValueAsJava(keySchema, HoodieRecord.PARTITION_PATH_METADATA_FIELD, CollectionUtils.emptyProps())); @@ -108,12 +112,12 @@ record -> { @Override public HoodieSchema readSchema(HoodieStorage storage, StoragePath filePath) { try (HoodieFileReader fileReader = - HoodieIOFactory.getIOFactory(storage) - .getReaderFactory(HoodieRecord.HoodieRecordType.SPARK) - .getFileReader( - ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER, - filePath, - HoodieFileFormat.LANCE)) { + HoodieIOFactory.getIOFactory(storage) + .getReaderFactory(HoodieRecord.HoodieRecordType.SPARK) + .getFileReader( + ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER, + filePath, + HoodieFileFormat.LANCE)) { return fileReader.getSchema(); } catch (IOException e) { throw new HoodieIOException("Failed to read schema from Lance file", e); @@ -137,18 +141,40 @@ public List readAvroRecords(HoodieStorage storage, StoragePath fi @Override public Map readFooter(HoodieStorage storage, boolean required, StoragePath filePath, String... footerNames) { - throw new UnsupportedOperationException("readFooter is not yet supported for Lance format"); + Map footerVals = new HashMap<>(); + long metadataAllocatorSize = Long.parseLong( + HoodieStorageConfig.LANCE_READ_METADATA_ALLOCATOR_SIZE_BYTES.defaultValue()); + try (BufferAllocator allocator = HoodieArrowAllocator.newChildAllocator( + getClass().getSimpleName() + "-metadata-" + filePath.getName(), metadataAllocatorSize); + LanceFileReader reader = LanceFileReader.open(filePath.toString(), allocator)) { + Map metadata = reader.schema().getCustomMetadata(); + for (String footerName : footerNames) { + if (metadata != null && metadata.containsKey(footerName)) { + footerVals.put(footerName, metadata.get(footerName)); + } else if (required) { + throw new MetadataNotFoundException( + "Could not find metadata key in Lance footer. Key " + footerName + ", file: " + filePath); + } + } + return footerVals; + } catch (MetadataNotFoundException e) { + throw e; + } catch (IOException e) { + throw new HoodieIOException("Unable to read footer for Lance: " + filePath, e); + } catch (Exception e) { + throw new HoodieException("Unable to close Lance footer reader: " + filePath, e); + } } @Override public long getRowCount(HoodieStorage storage, StoragePath filePath) { try (HoodieFileReader fileReader = - HoodieIOFactory.getIOFactory(storage) - .getReaderFactory(HoodieRecord.HoodieRecordType.SPARK) - .getFileReader( - ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER, - filePath, - HoodieFileFormat.LANCE)) { + HoodieIOFactory.getIOFactory(storage) + .getReaderFactory(HoodieRecord.HoodieRecordType.SPARK) + .getFileReader( + ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER, + filePath, + HoodieFileFormat.LANCE)) { return fileReader.getTotalRecords(); } catch (IOException e) { throw new HoodieIOException("Failed to read schema from Lance file", e); @@ -158,12 +184,12 @@ public long getRowCount(HoodieStorage storage, StoragePath filePath) { @Override public Set> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set filterKeys) { try (HoodieFileReader fileReader = - HoodieIOFactory.getIOFactory(storage) - .getReaderFactory(HoodieRecord.HoodieRecordType.SPARK) - .getFileReader( - ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER, - filePath, - HoodieFileFormat.LANCE)) { + HoodieIOFactory.getIOFactory(storage) + .getReaderFactory(HoodieRecord.HoodieRecordType.SPARK) + .getFileReader( + ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER, + filePath, + HoodieFileFormat.LANCE)) { return fileReader.filterRowKeys(filterKeys); } catch (IOException e) { throw new HoodieIOException("Failed to read filter keys from Lance file", e); diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java index 2700405073078..00708a5e97ec3 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java @@ -28,7 +28,6 @@ import org.apache.hudi.common.util.collection.ImmutablePair; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.exception.InvalidHoodiePathException; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; @@ -56,11 +55,8 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.regex.Matcher; import java.util.stream.Collectors; -import static org.apache.hudi.common.fs.FSUtils.LOG_FILE_PATTERN; - /** * Utility functions related to accessing the file storage on Hadoop. */ @@ -397,22 +393,14 @@ public static String getRelativePartitionPath(Path basePath, Path fullPartitionP * the file name. */ public static String getFileIdFromLogPath(Path path) { - Matcher matcher = LOG_FILE_PATTERN.matcher(path.getName()); - if (!matcher.matches()) { - throw new InvalidHoodiePathException(path.toString(), "LogFile"); - } - return matcher.group(1); + return FSUtils.getFileIdFromLogPath(new StoragePath(path.toUri())); } /** * Get the second part of the file name in the log file. That will be the delta commit time. */ public static String getDeltaCommitTimeFromLogPath(Path path) { - Matcher matcher = LOG_FILE_PATTERN.matcher(path.getName()); - if (!matcher.matches()) { - throw new InvalidHoodiePathException(path.toString(), "LogFile"); - } - return matcher.group(2); + return FSUtils.getDeltaCommitTimeFromLogPath(new StoragePath(path.toUri())); } /** @@ -426,8 +414,7 @@ public static String getFileIdFromFilePath(Path filePath) { } public static boolean isBaseFile(Path path) { - String extension = FSUtils.getFileExtension(path.getName()); - return HoodieFileFormat.BASE_FILE_EXTENSIONS.contains(extension); + return FSUtils.isBaseFile(new StoragePath(path.toUri())); } public static boolean isLogFile(Path logPath) { diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/lance/HoodieBaseLanceWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/lance/HoodieBaseLanceWriter.java index c2363ab3a59ba..be0a94216af23 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/lance/HoodieBaseLanceWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/lance/HoodieBaseLanceWriter.java @@ -38,6 +38,8 @@ import java.io.Closeable; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; /** @@ -68,6 +70,7 @@ public abstract class HoodieBaseLanceWriter> implemen private VectorSchemaRoot root; private ArrowWriter arrowWriter; protected final Option> bloomFilterWriteSupportOpt; + private final Map footerMetadata = new LinkedHashMap<>(); private LanceFileWriter writer; @@ -124,6 +127,19 @@ protected Map additionalSchemaMetadata() { return Collections.emptyMap(); } + /** + * Add Hudi file footer metadata to the Lance schema custom metadata. + * + *

Lance exposes file-level key-value metadata through the Arrow schema custom metadata + * returned by {@code LanceFileReader.schema()}. The metadata is flushed to the file footer + * when the underlying {@link LanceFileWriter} is closed. + */ + public void addFooterMetadata(Map footerMetadata) { + if (footerMetadata != null && !footerMetadata.isEmpty()) { + this.footerMetadata.putAll(footerMetadata); + } + } + /** * Write a single record. Records are buffered and flushed in batches. * @@ -216,6 +232,10 @@ public void close() throws IOException { if (extra != null && !extra.isEmpty()) { writer.addSchemaMetadata(extra); } + + if (!footerMetadata.isEmpty()) { + writer.addSchemaMetadata(new HashMap<>(footerMetadata)); + } } } catch (Exception e) { primaryException = e; diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcReader.java index 188081b86ec68..3d981335d9cb9 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcReader.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcReader.java @@ -51,7 +51,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -87,10 +86,6 @@ public Set> filterRowKeys(Set candidateRowKeys) { @Override public ClosableIterator getIndexedRecordIterator(HoodieSchema readerSchema, HoodieSchema requestedSchema, Map renamedColumns) { - if (!Objects.equals(readerSchema, requestedSchema)) { - throw new UnsupportedOperationException("Schema projections are not supported in HFile reader"); - } - Configuration hadoopConf = storage.getConf().unwrapCopyAs(Configuration.class); try (Reader reader = OrcFile.createReader(new Path(path.toUri()), OrcFile.readerOptions(hadoopConf))) { // Limit the ORC schema to requested fields only diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcWriter.java index 16abaef1c8630..4af106056cda5 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcWriter.java @@ -48,7 +48,9 @@ import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; @@ -68,6 +70,7 @@ public class HoodieAvroOrcWriter implements HoodieAvroFileWriter, Closeable { private final Option wrapperFs; private final String instantTime; private final TaskContextSupplier taskContextSupplier; + private final Map footerMetadata = new LinkedHashMap<>(); private final HoodieOrcConfig orcConfig; private String minRecordKey; @@ -151,6 +154,11 @@ public void writeAvro(String recordKey, IndexedRecord object) throws IOException } } + @Override + public void addFooterMetadata(Map footerMetadata) { + this.footerMetadata.putAll(footerMetadata); + } + @Override public void close() throws IOException { if (batch.size != 0) { @@ -170,6 +178,9 @@ public void close() throws IOException { } } writer.addUserMetadata(HoodieOrcConfig.AVRO_SCHEMA_METADATA_KEY, ByteBuffer.wrap(getUTF8Bytes(schema.toString()))); + for (Map.Entry entry : footerMetadata.entrySet()) { + writer.addUserMetadata(entry.getKey(), ByteBuffer.wrap(getUTF8Bytes(entry.getValue()))); + } writer.close(); } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index acb7056c2359c..1eb6391489f92 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -280,6 +280,27 @@ public void testCdcLogFileName() { assertEquals(2, FSUtils.getFileVersionFromLog(path)); } + @Test + public void testNativeParquetLogFileName() { + String partitionPath = "2022/11/04/"; + String fileName = UUID.randomUUID().toString(); + String logFile = String.format("%s_1-0-1_100_2.log.parquet", fileName); + StoragePath storagePath = new StoragePath(new StoragePath(partitionPath), logFile); + Path hadoopPath = new Path(storagePath.toString()); + + assertTrue(FSUtils.isLogFile(storagePath)); + assertEquals("log", FSUtils.getFileExtensionFromLog(storagePath)); + assertEquals(fileName, FSUtils.getFileIdFromLogPath(storagePath)); + assertEquals("100", FSUtils.getDeltaCommitTimeFromLogPath(storagePath)); + assertEquals(fileName, HadoopFSUtils.getFileIdFromLogPath(hadoopPath)); + assertEquals("100", HadoopFSUtils.getDeltaCommitTimeFromLogPath(hadoopPath)); + assertEquals(1, FSUtils.getTaskPartitionIdFromLogPath(storagePath)); + assertEquals("1-0-1", FSUtils.getWriteTokenFromLogPath(storagePath)); + assertEquals(0, FSUtils.getStageIdFromLogPath(storagePath)); + assertEquals(1, FSUtils.getTaskAttemptIdFromLogPath(storagePath)); + assertEquals(2, FSUtils.getFileVersionFromLog(storagePath)); + } + @Test public void testArchiveLogFileName() { String partitionPath = "2022/11/04/"; diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java index 65f1da8a48bcd..fbe0a9d63c10d 100755 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.functional; import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.common.config.RecordMergeMode; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.DeleteRecord; import org.apache.hudi.common.model.HoodieArchivedLogFile; @@ -57,9 +58,11 @@ import org.apache.hudi.common.testutils.FileCreateUtilsLegacy; import org.apache.hudi.common.testutils.HadoopMapRedUtils; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.testutils.minicluster.HdfsTestService; +import org.apache.hudi.common.testutils.reader.HoodieFileSliceTestUtils; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; @@ -180,7 +183,7 @@ public void setUp(TestInfo testInfo) throws IOException, InterruptedException { partitionPath = new StoragePath(basePath, "partition_path"); spillableBasePath = new StoragePath(workDir.toString(), ".spillable_path").toString(); assertTrue(storage.createDirectory(partitionPath)); - HoodieTestUtils.init(storage.getConf().newInstance(), basePath, HoodieTableType.MERGE_ON_READ); + metaClient = HoodieTestUtils.init(storage.getConf().newInstance(), basePath, HoodieTableType.MERGE_ON_READ); } @AfterEach @@ -490,7 +493,7 @@ public void testBasicWriteAndScan(int tableVersion) throws IOException, URISynta writer.appendBlock(dataBlock); writer.close(); - Reader reader = HoodieLogFormat.newReader(storage, writer.getLogFile(), SchemaTestUtil.getSimpleSchema()); + Reader reader = HoodieLogFormat.newReader(metaClient, writer.getLogFile(), SchemaTestUtil.getSimpleSchema()); assertTrue(reader.hasNext(), "We wrote a block, we should be able to read it"); HoodieLogBlock nextBlock = reader.next(); assertEquals(DEFAULT_DATA_BLOCK_TYPE, nextBlock.getBlockType(), "The next block should be a data block"); @@ -539,7 +542,7 @@ public void testHugeLogFileWrite() throws IOException, URISyntaxException, Inter } writer.close(); - Reader reader = HoodieLogFormat.newReader(storage, writer.getLogFile(), SchemaTestUtil.getSimpleSchema(), true); + Reader reader = HoodieLogFormat.newReader(metaClient, writer.getLogFile(), SchemaTestUtil.getSimpleSchema(), true); assertTrue(reader.hasNext(), "We wrote a block, we should be able to read it"); HoodieLogBlock nextBlock = reader.next(); assertEquals(DEFAULT_DATA_BLOCK_TYPE, nextBlock.getBlockType(), "The next block should be a data block"); @@ -617,7 +620,7 @@ public void testBasicAppendAndRead(HoodieLogBlockType dataBlockType) throws IOEx writer.close(); Reader reader = - HoodieLogFormat.newReader(storage, writer.getLogFile(), SchemaTestUtil.getSimpleSchema()); + HoodieLogFormat.newReader(metaClient, writer.getLogFile(), SchemaTestUtil.getSimpleSchema()); assertTrue(reader.hasNext(), "First block should be available"); HoodieLogBlock nextBlock = reader.next(); HoodieDataBlock dataBlockRead = (HoodieDataBlock) nextBlock; @@ -700,7 +703,7 @@ public void testCDCBlock() throws IOException, InterruptedException { writer.appendBlock(dataBlock); writer.close(); - Reader reader = HoodieLogFormat.newReader(storage, writer.getLogFile(), cdcSchema); + Reader reader = HoodieLogFormat.newReader(metaClient, writer.getLogFile(), cdcSchema); assertTrue(reader.hasNext()); HoodieLogBlock block = reader.next(); HoodieDataBlock dataBlockRead = (HoodieDataBlock) block; @@ -805,6 +808,87 @@ private void testBasicAppendAndScanMultipleFiles(ExternalSpillableMap.DiskMapTyp scanner.close(); } + @Test + public void testMergedLogRecordScannerReadsNativeDataAndDeleteLogFiles() throws IOException { + HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(basePath); + Properties updatedProps = new Properties(); + updatedProps.setProperty(HoodieTableConfig.RECORD_MERGE_MODE.key(), RecordMergeMode.EVENT_TIME_ORDERING.name()); + updatedProps.setProperty(HoodieTableConfig.ORDERING_FIELDS.key(), "timestamp"); + HoodieTableConfig.update(metaClient.getStorage(), metaClient.getMetaPath(), updatedProps); + + // Native data log file holding live records for key1, key2 and key3. + List dataRecords = new ArrayList<>(); + for (String key : Arrays.asList("key1", "key2", "key3")) { + IndexedRecord dataRecordWithoutMetaFields = HoodieFileSliceTestUtils.DATA_GEN.generateGenericRecord( + key, "partition_path", "rider-100", "driver-100", 5L); + GenericRecord dataRecord = HoodieAvroUtils.rewriteRecordWithNewSchema( + dataRecordWithoutMetaFields, HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS); + HoodieAvroUtils.addHoodieKeyToRecord(dataRecord, key, "partition_path", "test-fileid1"); + HoodieAvroUtils.addCommitMetadataToRecord(dataRecord, "100", "100_0_0"); + dataRecords.add(dataRecord); + } + + String nativeDataLogPath = new StoragePath(partitionPath, String.format("%s_%s_%s_%d%s", + "test-fileid1", "1-0-1", "100", 1, ".log.parquet")).toString(); + HoodieLogFile nativeDataLog = HoodieFileSliceTestUtils.createNativeDataLogFile( + nativeDataLogPath, + dataRecords, + HoodieTestDataGenerator.HOODIE_SCHEMA_WITH_METADATA_FIELDS, + "100"); + + // Native delete log file removing key2 with a higher ordering value, so the delete wins. + IndexedRecord deleteRecord = HoodieFileSliceTestUtils.DATA_GEN.generatePayloadForTripSchema( + new HoodieKey("key2", "partition_path"), "100", 7L); + HoodieLogFile nativeDeleteLog = HoodieFileSliceTestUtils.createNativeDeleteLogFile( + storage, + new StoragePath(partitionPath, String.format("%s_%s_%s_%d%s", + "test-fileid1", "1-0-1", "100", 2, ".deletes.parquet")).toString(), + Collections.singletonList(deleteRecord), + HoodieTestDataGenerator.HOODIE_SCHEMA, + "100"); + + FileCreateUtilsLegacy.createDeltaCommit(basePath, "100", storage); + + try (HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder() + .withStorage(storage) + .withBasePath(basePath) + .withLogFilePaths(Arrays.asList( + nativeDataLog.getPath().toString(), nativeDeleteLog.getPath().toString())) + .withReaderSchema(HoodieTestDataGenerator.HOODIE_SCHEMA_WITH_METADATA_FIELDS) + .withLatestInstantTime("100") + .withMaxMemorySizeInBytes(1024000L) + .withReverseReader(false) + .withBufferSize(BUFFER_SIZE) + .withSpillableMapBasePath(spillableBasePath) + .build()) { + Map recordsByKey = new HashMap<>(); + scanner.forEach(record -> recordsByKey.put(record.getRecordKey(), record)); + + // key1 and key3 survive as data records, key2 is retained as a delete. + assertEquals(3, recordsByKey.size()); + + for (String key : Arrays.asList("key1", "key3")) { + HoodieRecord dataResult = recordsByKey.get(key); + assertEquals("partition_path", dataResult.getPartitionPath()); + Object data = dataResult.toIndexedRecord( + HoodieTestDataGenerator.HOODIE_SCHEMA_WITH_METADATA_FIELDS, CollectionUtils.emptyProps()).get().getData(); + GenericRecord scannedRecord = data instanceof SerializableIndexedRecord + ? (GenericRecord) ((SerializableIndexedRecord) data).getData() : (GenericRecord) data; + assertEquals(key, scannedRecord.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString()); + assertEquals("partition_path", scannedRecord.get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString()); + assertEquals(5L, scannedRecord.get("timestamp")); + } + + // key2 is removed by the delete log file and resolves to a delete. + HoodieRecord deleteResult = recordsByKey.get("key2"); + assertEquals("partition_path", deleteResult.getPartitionPath()); + assertEquals(7L, deleteResult.getOrderingValue( + HoodieTestDataGenerator.HOODIE_SCHEMA, updatedProps, new String[] {"timestamp"})); + assertFalse(((HoodieRecordPayload) deleteResult.getData()) + .getInsertValue(HoodieTestDataGenerator.AVRO_SCHEMA).isPresent()); + } + } + @ParameterizedTest @MethodSource("testArguments") public void testBasicAppendAndPartialScanning(ExternalSpillableMap.DiskMapType diskMapType, @@ -997,7 +1081,7 @@ public void testAppendAndReadOnCorruptedLog() throws IOException, URISyntaxExcep logFile = appendValidBlock(logFile.getPath(), "test-fileId1", "100", 10); // First round of reads - we should be able to read the first block and then EOF - Reader reader = HoodieLogFormat.newReader(storage, logFile, SchemaTestUtil.getSimpleSchema()); + Reader reader = HoodieLogFormat.newReader(metaClient, logFile, SchemaTestUtil.getSimpleSchema()); assertTrue(reader.hasNext(), "First block should be available"); reader.next(); assertTrue(reader.hasNext(), "We should have corrupted block next"); @@ -1028,7 +1112,7 @@ public void testAppendAndReadOnCorruptedLog() throws IOException, URISyntaxExcep logFile = appendValidBlock(logFile.getPath(), "test-fileId1", "100", 100); // Second round of reads - we should be able to read the first and last block - reader = HoodieLogFormat.newReader(storage, logFile, SchemaTestUtil.getSimpleSchema()); + reader = HoodieLogFormat.newReader(metaClient, logFile, SchemaTestUtil.getSimpleSchema()); assertTrue(reader.hasNext(), "First block should be available"); reader.next(); assertTrue(reader.hasNext(), "We should get the 1st corrupted block next"); @@ -1084,7 +1168,7 @@ public void testMissingBlockExceptMagicBytes() throws IOException, URISyntaxExce logFile = appendValidBlock(logFile.getPath(), "test-fileId1", "100", 10); // First round of reads - we should be able to read the first block and then EOF - Reader reader = HoodieLogFormat.newReader(storage, logFile, SchemaTestUtil.getSimpleSchema()); + Reader reader = HoodieLogFormat.newReader(metaClient, logFile, SchemaTestUtil.getSimpleSchema()); assertTrue(reader.hasNext(), "First block should be available"); reader.next(); assertTrue(reader.hasNext(), "We should have corrupted block next"); @@ -1176,7 +1260,7 @@ public void testValidateCorruptBlockEndPosition() throws IOException, URISyntaxE appendValidBlock(writer.getLogFile().getPath(), "test-fileid1", "100", 10); // Read data and corrupt block - Reader reader = HoodieLogFormat.newReader(storage, writer.getLogFile(), SchemaTestUtil.getSimpleSchema()); + Reader reader = HoodieLogFormat.newReader(metaClient, writer.getLogFile(), SchemaTestUtil.getSimpleSchema()); assertTrue(reader.hasNext(), "First block should be available"); reader.next(); assertTrue(reader.hasNext(), "We should have corrupted block next"); @@ -2873,7 +2957,7 @@ public void testDataBlockFormatAppendAndReadWithProjectedSchema( List projectedRecords = HoodieAvroUtils.rewriteRecords(records, projectedSchema.toAvroSchema()); - try (Reader reader = HoodieLogFormat.newReader(storage, writer.getLogFile(), projectedSchema, false)) { + try (Reader reader = HoodieLogFormat.newReader(metaClient, writer.getLogFile(), projectedSchema, false)) { assertTrue(reader.hasNext(), "First block should be available"); HoodieLogBlock nextBlock = reader.next(); @@ -3018,7 +3102,7 @@ private HoodieLogFormat.Reader createCorruptedFile(String fileId) throws Excepti outputStream.close(); // First round of reads - we should be able to read the first block and then EOF - Reader reader = HoodieLogFormat.newReader(storage, writer.getLogFile(), SchemaTestUtil.getSimpleSchema()); + Reader reader = HoodieLogFormat.newReader(metaClient, writer.getLogFile(), SchemaTestUtil.getSimpleSchema()); assertTrue(reader.hasNext(), "First block should be available"); reader.next(); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java index 00e50e38b335e..6cc7aff450a58 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java @@ -127,9 +127,8 @@ void testReadSchemaFromLogFile() throws IOException, URISyntaxException, Interru StoragePath partitionPath = new StoragePath(testDir, "partition1"); HoodieSchema expectedSchema = getSimpleSchema(); StoragePath logFilePath = writeLogFile(partitionPath, expectedSchema.toAvroSchema()); - assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile( - HoodieStorageUtils.getStorage(new StoragePath(logFilePath.toString()), HoodieTestUtils.getDefaultStorageConf()), - logFilePath)); + HoodieTableMetaClient metaClient = HoodieTestUtils.init(HoodieTestUtils.getDefaultStorageConf(), testDir); + assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile(metaClient, logFilePath)); } @Test @@ -147,9 +146,8 @@ void testReadSchemaFromNativeDataLogFile() throws IOException { expectedSchema, "100"); - assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile( - HoodieStorageUtils.getStorage(logFilePath, HoodieTestUtils.getDefaultStorageConf()), - logFilePath)); + HoodieTableMetaClient metaClient = HoodieTestUtils.init(HoodieTestUtils.getDefaultStorageConf(), testDir); + assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile(metaClient, logFilePath)); } @Test @@ -169,13 +167,16 @@ void testReadSchemaFromNativeDeleteLogFileReturnsNull() throws IOException { tableSchema, "100"); - assertNull(TableSchemaResolver.readSchemaFromLogFile(storage, logFilePath)); + HoodieTableMetaClient metaClient = HoodieTestUtils.init(HoodieTestUtils.getDefaultStorageConf(), testDir); + assertNull(TableSchemaResolver.readSchemaFromLogFile(metaClient, logFilePath)); } @Test void testReadSchemaFromNativeDataLogFileRequiresSchemaFooter() { StoragePath logFilePath = new StoragePath("/tmp/test-fileid1_1-0-1_100_1.log.parquet"); + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); HoodieStorage storage = mock(HoodieStorage.class); + when(metaClient.getStorage()).thenReturn(storage); HoodieIOFactory ioFactory = mock(HoodieIOFactory.class); FileFormatUtils fileFormatUtils = mock(FileFormatUtils.class); when(ioFactory.getFileFormatUtils(HoodieFileFormat.PARQUET)).thenReturn(fileFormatUtils); @@ -186,7 +187,7 @@ void testReadSchemaFromNativeDataLogFileRequiresSchemaFooter() { ioFactoryMockedStatic.when(() -> HoodieIOFactory.getIOFactory(storage)).thenReturn(ioFactory); HoodieIOException exception = assertThrows(HoodieIOException.class, - () -> TableSchemaResolver.readSchemaFromLogFile(storage, logFilePath)); + () -> TableSchemaResolver.readSchemaFromLogFile(metaClient, logFilePath)); assertTrue(exception.getMessage().contains(HoodieLogBlock.HeaderMetadataType.SCHEMA.name())); assertTrue(exception.getMessage().contains(NativeLogFooterMetadata.FOOTER_METADATA_KEY)); assertTrue(exception.getMessage().contains(logFilePath.toString())); @@ -239,7 +240,7 @@ void testHasOperationFieldFileInspectionOrdering() throws IOException { when(fileFormatUtils.readSchema(any(), eq(parquetPath))).thenReturn(null); ioFactoryMockedStatic.when(() -> HoodieIOFactory.getIOFactory(any())).thenReturn(ioFactory); // mock log file schema reading to return the expected schema - tableSchemaResolverMockedStatic.when(() -> TableSchemaResolver.readSchemaFromLogFile(any(), eq(new StoragePath("/tmp/hudi_table/" + logFileWriteStat.getPath())))) + tableSchemaResolverMockedStatic.when(() -> TableSchemaResolver.readSchemaFromLogFile(eq(metaClient), eq(new StoragePath("/tmp/hudi_table/" + logFileWriteStat.getPath())))) .thenReturn(schema); assertTrue(schemaResolver.hasOperationField()); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderNativeLogs.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderNativeLogs.java index cf1990710715d..8ef00296d52ae 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderNativeLogs.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderNativeLogs.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieBaseFile; +import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableConfig; @@ -106,6 +107,22 @@ public void testPureNativeLogFiles() throws IOException { Arrays.asList(4L, 2L, 4L)); } + @Test + public void testNativeHFileLogFile() throws IOException { + metaClient.getTableConfig().setValue(HoodieTableConfig.BASE_FILE_FORMAT, HoodieFileFormat.HFILE.name()); + readerContext = new HoodieAvroReaderContext( + storageConf, metaClient.getTableConfig(), Option.empty(), Option.empty()); + preparePartitionPath(); + + HoodieLogFile nativeDataLog = createNativeDataLogFile( + "001", 1, records(Arrays.asList("1", "2", "3"), 2L, false), ".log.hfile"); + FileSlice fileSlice = createFileSlice(null, nativeDataLog); + + assertFileGroupRecords(fileSlice, + Arrays.asList("1", "2", "3"), + Arrays.asList(2L, 2L, 2L)); + } + @Test public void testNativeLogFilesWithKeyFilters() throws IOException { preparePartitionPath(); @@ -129,6 +146,29 @@ public void testNativeLogFilesWithKeyFilters() throws IOException { Arrays.asList(2L, 4L)); } + @Test + public void testNativeHFileLogFilesWithKeyFilters() throws IOException { + preparePartitionPath(); + + HoodieLogFile nativeDataLog1 = createNativeHFileDataLogFile("001", 1, records(Arrays.asList("1", "2", "30"), 2L, false)); + HoodieLogFile nativeDataLog2 = createNativeHFileDataLogFile("002", 2, records(Arrays.asList("1", "4", "31"), 4L, false)); + FileSlice fileSlice = createFileSlice(null, nativeDataLog1, nativeDataLog2); + + Predicate inPredicate = Predicates.in( + Literal.from(ROW_KEY), + Arrays.asList(Literal.from("1"), Literal.from("31"))); + assertFileGroupRecords(fileSlice, inPredicate, + Arrays.asList("1", "31"), + Arrays.asList(4L, 4L)); + + Predicate prefixPredicate = Predicates.startsWithAny( + Literal.from(ROW_KEY), + Arrays.asList(Literal.from("3"))); + assertFileGroupRecords(fileSlice, prefixPredicate, + Arrays.asList("30", "31"), + Arrays.asList(2L, 4L)); + } + @Test public void testBaseFileWithNativeLogFiles() throws IOException { preparePartitionPath(); @@ -262,8 +302,22 @@ private HoodieBaseFile createBaseFile(String instantTime, List re private HoodieLogFile createNativeDataLogFile(String instantTime, int version, List records) throws IOException { + return createNativeDataLogFile(instantTime, version, records, ".log.parquet"); + } + + private HoodieLogFile createNativeDataLogFile(String instantTime, int version, List records, String extension) + throws IOException { return HoodieFileSliceTestUtils.createNativeDataLogFile( - partitionBasePath() + "/" + nativeLogFileName(instantTime, version, ".log.parquet"), + partitionBasePath() + "/" + nativeLogFileName(instantTime, version, extension), + records, + HOODIE_SCHEMA, + instantTime); + } + + private HoodieLogFile createNativeHFileDataLogFile(String instantTime, int version, List records) + throws IOException { + return HoodieFileSliceTestUtils.createNativeHFileDataLogFile( + partitionBasePath() + "/" + nativeLogFileName(instantTime, version, ".log.hfile"), records, HOODIE_SCHEMA, instantTime); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java index d576087bb3f13..4b44c3fd6cdca 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java @@ -51,7 +51,6 @@ import org.apache.hudi.common.util.CompactionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; -import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathFilter; @@ -446,7 +445,7 @@ void readLegacyArchivedTimelineWithNullFields() { @ValueSource(strings = {"hudi_0_13_0_archive", "hudi_0_8_0_archive"}) void shouldReadArchivedFileAndValidateContent(String archivePath) { String path = this.getClass().getClassLoader().getResource(archivePath).getPath() + "/.commits_.archive.1_1-0-1"; - assertDoesNotThrow(() -> readAndValidateArchivedFile(path, metaClient.getStorage())); + assertDoesNotThrow(() -> readAndValidateArchivedFile(path)); } @Test @@ -525,9 +524,9 @@ public void testReadArchivedTimelineWithLeftoverMergedArchiveFileDoesNotDuplicat assertTrue(archivedTimeline.getInstantDetails(commit05).isPresent(), "commit metadata for ts=05 must be loaded"); } - private void readAndValidateArchivedFile(String path, HoodieStorage storage) throws IOException { + private void readAndValidateArchivedFile(String path) throws IOException { try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader( - storage, new HoodieLogFile(path), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) { + metaClient, new HoodieLogFile(path), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) { while (reader.hasNext()) { HoodieLogBlock block = reader.next(); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java index 46b5248e6f337..b2b37ea47fea7 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java @@ -976,6 +976,8 @@ public List listAllBaseFiles() throws IOException { public List listAllBaseFiles(String fileExtension) throws IOException { return listRecursive(storage, new StoragePath(basePath)).stream() + .filter(fileInfo -> FSUtils.isBaseFile(fileInfo.getPath().getName())) + // filter out MDT base files. .filter(fileInfo -> fileInfo.getPath().getName().endsWith(fileExtension)) .collect(Collectors.toList()); } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java index 47eef35f11675..aa0e606bb9e93 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieAvroIndexedRecord; import org.apache.hudi.common.model.HoodieBaseFile; +import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieFileGroupId; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieLogFile; @@ -70,6 +71,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -356,6 +358,30 @@ public static HoodieLogFile createNativeDataLogFile( return new HoodieLogFile(logFilePath); } + public static HoodieLogFile createNativeHFileDataLogFile( + String logFilePath, + List records, + HoodieSchema schema, + String logInstantTime + ) throws IOException { + HoodieStorage storage = HoodieTestUtils.getStorage(logFilePath); + HoodieConfig config = HoodieStorageConfig.newBuilder().build(); + config.setValue(HoodieTableConfig.POPULATE_META_FIELDS.key(), "true"); + try (HoodieAvroFileWriter writer = (HoodieAvroFileWriter) HoodieFileWriterFactory.getFileWriter( + logInstantTime, new StoragePath(logFilePath), storage, config, schema, + new LocalTaskContextSupplier(), HoodieRecord.HoodieRecordType.AVRO)) { + writer.addFooterMetadata(NativeLogFooterMetadata.toFooterMetadata(getNativeLogBlockHeader(schema, logInstantTime))); + List sortedRecords = records.stream() + .sorted(Comparator.comparing(record -> record.get(schema.getField(ROW_KEY).get().pos()).toString())) + .collect(Collectors.toList()); + for (IndexedRecord record : sortedRecords) { + writer.writeAvro( + (String) record.get(schema.getField(ROW_KEY).get().pos()), record); + } + } + return new HoodieLogFile(logFilePath); + } + public static HoodieLogFile createNativeDeleteLogFile( HoodieStorage storage, String logFilePath, @@ -382,6 +408,21 @@ private static HoodieAvroFileWriter createNativeLogWriter( HoodieSchema schema, String logInstantTime ) throws IOException { + if (logFilePath.endsWith(HoodieFileFormat.HFILE.getFileExtension())) { + Properties props = new Properties(); + props.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(), Boolean.FALSE.toString()); + HoodieAvroFileWriter writer = (HoodieAvroFileWriter) HoodieFileWriterFactory.getFileWriter( + logInstantTime, + new StoragePath(logFilePath), + storage, + HoodieStorageConfig.newBuilder().fromProperties(props).build(), + schema, + new LocalTaskContextSupplier(), + HoodieRecord.HoodieRecordType.AVRO); + writer.addFooterMetadata(NativeLogFooterMetadata.toFooterMetadata(getNativeLogBlockHeader(schema, logInstantTime))); + return writer; + } + HoodieAvroWriteSupport writeSupport = new HoodieAvroWriteSupport( getAvroSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(schema), schema, diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java index 1f0caed30abd2..6f61cb5b24b04 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java @@ -463,6 +463,39 @@ public void testReaderGetRecordIterator(boolean useBloomFilter) throws Exception } } + @Test + public void testReaderGetRecordIteratorWithProjectedSchema() throws Exception { + writeFileWithSimpleSchema(); + HoodieSchema dataSchema = + getHoodieSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc"); + // Mirrors metadata-table reads where Spark pushes a subset of columns to the HFile reader. + HoodieSchema projectedSchema = new HoodieSchema.Parser().parse( + "{" + + "\"namespace\":\"example.schema\"," + + "\"type\":\"record\"," + + "\"name\":\"trip\"," + + "\"fields\":[" + + "{\"name\":\"time\",\"type\":\"string\"}," + + "{\"name\":\"number\",\"type\":[\"int\",\"null\"]}" + + "]" + + "}"); + + try (HoodieAvroHFileReaderImplBase hfileReader = + createReader(HoodieTestUtils.getStorage(getFilePath()), true); + ClosableIterator> iterator = + hfileReader.getRecordIterator(dataSchema, projectedSchema)) { + int index = 0; + while (iterator.hasNext()) { + GenericRecord record = (GenericRecord) iterator.next().getData(); + assertNull(record.getSchema().getField("_row_key")); + assertEquals(Integer.toString(index), record.get("time").toString()); + assertEquals(index, record.get("number")); + index++; + } + assertEquals(NUM_RECORDS, index); + } + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testReaderGetRecordIteratorByKeys(boolean useBloomFilter) throws Exception { diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java index fc231fdb26fee..f1d29fed91e8d 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java @@ -21,6 +21,7 @@ import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.util.HFileUtils; +import org.apache.hudi.common.util.LanceUtils; import org.apache.hudi.common.util.OrcUtils; import org.apache.hudi.common.util.ParquetUtils; import org.apache.hudi.core.io.storage.HoodieIOFactory; @@ -51,6 +52,7 @@ public void testGetFileFormatUtils() throws IOException { assertTrue(ioFactory.getFileFormatUtils(new StoragePath("file:///a/b.parquet")) instanceof ParquetUtils); assertTrue(ioFactory.getFileFormatUtils(new StoragePath("file:///a/b.orc")) instanceof OrcUtils); assertTrue(ioFactory.getFileFormatUtils(new StoragePath("file:///a/b.hfile")) instanceof HFileUtils); + assertTrue(ioFactory.getFileFormatUtils(new StoragePath("file:///a/b.lance")) instanceof LanceUtils); assertThrows( UnsupportedOperationException.class, () -> ioFactory.getFileFormatUtils(new StoragePath("file:///a/b.log"))); @@ -58,6 +60,7 @@ public void testGetFileFormatUtils() throws IOException { assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.PARQUET) instanceof ParquetUtils); assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.ORC) instanceof OrcUtils); assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.HFILE) instanceof HFileUtils); + assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.LANCE) instanceof LanceUtils); assertThrows( UnsupportedOperationException.class, () -> ioFactory.getFileFormatUtils(HoodieFileFormat.HOODIE_LOG)); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java index d63143488c854..8180c64951914 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java @@ -24,11 +24,13 @@ import org.apache.hudi.common.bloom.BloomFilterFactory; import org.apache.hudi.common.bloom.BloomFilterTypeCode; import org.apache.hudi.common.config.HoodieStorageConfig; -import org.apache.hudi.common.engine.TaskContextSupplier; +import org.apache.hudi.common.engine.LocalTaskContextSupplier; +import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.core.io.storage.HoodieAvroFileReader; +import org.apache.hudi.core.io.storage.HoodieAvroFileWriter; import org.apache.hudi.core.io.storage.HoodieIOFactory; import org.apache.hudi.core.io.storage.HoodieOrcConfig; import org.apache.hudi.io.storage.hadoop.HoodieAvroOrcWriter; @@ -41,17 +43,18 @@ import org.apache.orc.CompressionKind; import org.apache.orc.OrcFile; import org.apache.orc.Reader; -import org.mockito.Mockito; +import org.junit.jupiter.api.Test; import java.io.IOException; -import java.util.function.Supplier; +import java.util.Collections; +import java.util.Map; import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY; +import static org.apache.hudi.common.testutils.SchemaTestUtil.getHoodieSchemaFromResource; import static org.apache.hudi.common.util.ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER; import static org.apache.hudi.core.io.storage.HoodieOrcConfig.AVRO_SCHEMA_METADATA_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; public class TestHoodieOrcReaderWriter extends TestHoodieReaderWriterBase { @@ -69,12 +72,8 @@ protected HoodieAvroOrcWriter createWriter( int orcBlockSize = Integer.parseInt(HoodieStorageConfig.ORC_BLOCK_SIZE.defaultValue()); int maxFileSize = Integer.parseInt(HoodieStorageConfig.ORC_FILE_MAX_SIZE.defaultValue()); HoodieOrcConfig config = new HoodieOrcConfig(conf, CompressionKind.ZLIB, orcStripSize, orcBlockSize, maxFileSize, filter); - TaskContextSupplier mockTaskContextSupplier = Mockito.mock(TaskContextSupplier.class); - Supplier partitionSupplier = Mockito.mock(Supplier.class); - when(mockTaskContextSupplier.getPartitionIdSupplier()).thenReturn(partitionSupplier); - when(partitionSupplier.get()).thenReturn(10); String instantTime = "000"; - return new HoodieAvroOrcWriter(instantTime, getFilePath(), config, schema, mockTaskContextSupplier); + return new HoodieAvroOrcWriter(instantTime, getFilePath(), config, schema, new LocalTaskContextSupplier()); } @Override @@ -109,4 +108,18 @@ protected void verifySchema(HoodieStorage storage, String schemaPath) throws IOE orcReader.getSchema().toString()); } } + + @Test + public void testAddFooterMetadata() throws Exception { + HoodieSchema schema = getHoodieSchemaFromResource(TestHoodieOrcReaderWriter.class, "/exampleSchema.avsc"); + try (HoodieAvroFileWriter writer = createWriter(schema, true)) { + writer.addFooterMetadata(Collections.singletonMap("custom.footer.key", "custom-footer-value")); + } + + HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath()); + Map footer = HoodieIOFactory.getIOFactory(storage) + .getFileFormatUtils(HoodieFileFormat.ORC) + .readFooter(storage, true, getFilePath(), "custom.footer.key"); + assertEquals("custom-footer-value", footer.get("custom.footer.key")); + } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveRecordContext.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveRecordContext.java index bc12027921f68..d38ff0a83d663 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveRecordContext.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveRecordContext.java @@ -100,6 +100,11 @@ public ArrayWritable constructEngineRecord(HoodieSchema recordSchema, Object[] f return new ArrayWritable(Writable.class, (Writable[]) fieldValues); } + @Override + public HoodieRecord.HoodieRecordType getEngineRecordType() { + return HoodieRecord.HoodieRecordType.HIVE; + } + @Override public ArrayWritable mergeWithEngineRecord(HoodieSchema schema, Map updateValues, diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala index 3281e9de653b6..c2a2adfca16bf 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala @@ -106,10 +106,10 @@ class ExportInstantsProcedure extends BaseProcedure with ProcedureBuilder with L numCopied = copyNonArchivedInstants(metaClient, nonArchivedInstants, numExports, localFolder) if (numCopied < numExports) { Collections.reverse(archivedStatuses) - numCopied += copyArchivedInstants(basePath, archivedStatuses, actionSet, numExports - numCopied, localFolder) + numCopied += copyArchivedInstants(metaClient, archivedStatuses, actionSet, numExports - numCopied, localFolder) } } else { - numCopied = copyArchivedInstants(basePath, archivedStatuses, actionSet, numExports, localFolder) + numCopied = copyArchivedInstants(metaClient, archivedStatuses, actionSet, numExports, localFolder) if (numCopied < numExports) numCopied += copyNonArchivedInstants(metaClient, nonArchivedInstants, numExports - numCopied, localFolder) } @@ -117,13 +117,13 @@ class ExportInstantsProcedure extends BaseProcedure with ProcedureBuilder with L } @throws[Exception] - private def copyArchivedInstants(basePath: String, statuses: util.List[FileStatus], actionSet: util.Set[String], limit: Int, localFolder: String) = { + private def copyArchivedInstants(metaClient: HoodieTableMetaClient, statuses: util.List[FileStatus], actionSet: util.Set[String], limit: Int, localFolder: String) = { var copyCount = 0 - val storage = HoodieStorageUtils.getStorage(basePath, HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration())) + val storage = metaClient.getStorage for (fs <- statuses.asScala) { // read the archived file val reader = HoodieLogFormat.newReader( - storage, new HoodieLogFile(convertToStoragePath(fs.getPath)), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema)) + metaClient, new HoodieLogFile(convertToStoragePath(fs.getPath)), HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema)) // read the avro blocks while ( { reader.hasNext && copyCount < limit diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala index 5257f3c20bda4..46e2f7b8b92a3 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala @@ -63,7 +63,8 @@ class ShowHoodieLogFileMetadataProcedure extends BaseProcedure with ProcedureBui val filter = getArgValueOrDefault(args, parameters(4)).get.asInstanceOf[String] validateFilter(filter, outputType) - val storage = createMetaClient(jsc, basePath).getStorage + val metaClient = createMetaClient(jsc, basePath) + val storage = metaClient.getStorage val logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(storage, new StoragePath(logFilePathPattern)).iterator().asScala .map(_.getPath.toString).toList val commitCountAndMetadata = @@ -73,8 +74,8 @@ class ShowHoodieLogFileMetadataProcedure extends BaseProcedure with ProcedureBui logFilePaths.foreach { logFilePath => { val statuses = storage.listDirectEntries(new StoragePath(logFilePath)) - val schema = TableSchemaResolver.readSchemaFromLogFile(storage, new StoragePath(logFilePath)) - val reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(statuses.get(0).getPath), schema) + val schema = TableSchemaResolver.readSchemaFromLogFile(metaClient, new StoragePath(logFilePath)) + val reader = HoodieLogFormat.newReader(metaClient, new HoodieLogFile(statuses.get(0).getPath), schema) // read the avro blocks while (reader.hasNext) { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala index 4b9e0e83c7d59..ad7873ff52f0c 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala @@ -70,7 +70,7 @@ class ShowHoodieLogFileRecordsProcedure extends BaseProcedure with ProcedureBuil ValidationUtils.checkArgument(logFilePaths.nonEmpty, "There is no log file") val allRecords: java.util.List[IndexedRecord] = new java.util.ArrayList[IndexedRecord] if (merge) { - val schema = Objects.requireNonNull(TableSchemaResolver.readSchemaFromLogFile(storage, new StoragePath(logFilePaths.last))) + val schema = Objects.requireNonNull(TableSchemaResolver.readSchemaFromLogFile(client, new StoragePath(logFilePaths.last))) val scanner = HoodieMergedLogRecordScanner.newBuilder .withStorage(storage) .withBasePath(basePath) @@ -93,8 +93,8 @@ class ShowHoodieLogFileRecordsProcedure extends BaseProcedure with ProcedureBuil } else { logFilePaths.toStream.takeWhile(_ => allRecords.size() < limit).foreach { logFilePath => { - val schema = Objects.requireNonNull(TableSchemaResolver.readSchemaFromLogFile(storage, new StoragePath(logFilePath))) - val reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(logFilePath), schema) + val schema = Objects.requireNonNull(TableSchemaResolver.readSchemaFromLogFile(client, new StoragePath(logFilePath))) + val reader = HoodieLogFormat.newReader(client, new HoodieLogFile(logFilePath), schema) while (reader.hasNext) { val block = reader.next() block match { diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java index 191fbfd645fbb..5117b21a1bd5d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBufferedRecordMerger.java @@ -958,6 +958,11 @@ public InternalRow constructEngineRecord(HoodieSchema recordSchema, Object[] fie return new GenericInternalRow(fieldValues); } + @Override + public HoodieRecord.HoodieRecordType getEngineRecordType() { + return HoodieRecord.HoodieRecordType.SPARK; + } + @Override public InternalRow mergeWithEngineRecord(HoodieSchema schema, Map updateValues, diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java index 5b2fb08a40d37..d71286b5849ee 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java @@ -504,7 +504,7 @@ public void testAutoDeletePartitionsDisabledPreservesColumnStats() throws Except // Step 5: Verify column stats log files contain valid column stats records List colStatsLogFiles = colStatsSlicesAfter.get(0) .getLogFiles().collect(Collectors.toList()); - verifyMetadataColumnStatsRecords(storage, colStatsLogFiles); + verifyMetadataColumnStatsRecords(metadataMetaClient, colStatsLogFiles); } @Test @@ -1612,14 +1612,14 @@ private static void verifyMetadataRawRecords(HoodieTable table, List pathInfoList = storage.listDirectEntries(logFile.getPath()); HoodieSchema writerSchema = - TableSchemaResolver.readSchemaFromLogFile(storage, logFile.getPath()); + TableSchemaResolver.readSchemaFromLogFile(table.getMetaClient(), logFile.getPath()); if (writerSchema == null) { // not a data block continue; } - try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage, - new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { + try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader( + table.getMetaClient(), new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { while (logFileReader.hasNext()) { HoodieLogBlock logBlock = logFileReader.next(); if (logBlock instanceof HoodieDataBlock) { @@ -4119,7 +4119,7 @@ private void validateMetadata(SparkRDDWriteClient testClient, Option ign verifyMetadataRawRecords(table, logFiles, false); } if (COLUMN_STATS.getPartitionPath().equals(partition)) { - verifyMetadataColumnStatsRecords(storage, logFiles); + verifyMetadataColumnStatsRecords(metadataMetaClient, logFiles); } } catch (IOException e) { log.error("Metadata record validation failed", e); @@ -4285,7 +4285,7 @@ public static void validateMetadata(HoodieWriteConfig config, Option ign verifyMetadataRawRecords(table, logFiles, false); } if (COLUMN_STATS.getPartitionPath().equals(partition)) { - verifyMetadataColumnStatsRecords(storage, logFiles); + verifyMetadataColumnStatsRecords(metadataMetaClient, logFiles); } } catch (Exception e) { log.error("Metadata record validation failed", e); @@ -4298,18 +4298,20 @@ public static void validateMetadata(HoodieWriteConfig config, Option ign } } - private static void verifyMetadataColumnStatsRecords(HoodieStorage storage, List logFiles) throws IOException { + private static void verifyMetadataColumnStatsRecords(HoodieTableMetaClient metadataMetaClient, + List logFiles) throws IOException { + HoodieStorage storage = metadataMetaClient.getStorage(); for (HoodieLogFile logFile : logFiles) { List pathInfoList = storage.listDirectEntries(logFile.getPath()); HoodieSchema writerSchema = - TableSchemaResolver.readSchemaFromLogFile(storage, logFile.getPath()); + TableSchemaResolver.readSchemaFromLogFile(metadataMetaClient, logFile.getPath()); if (writerSchema == null) { // not a data block continue; } - try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage, - new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { + try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader( + metadataMetaClient, new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) { while (logFileReader.hasNext()) { HoodieLogBlock logBlock = logFileReader.next(); if (logBlock instanceof HoodieDataBlock) { diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMergeHandle.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMergeHandle.java index 95a5d64cf0fb2..2e6db79cb3ef1 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMergeHandle.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMergeHandle.java @@ -18,7 +18,6 @@ package org.apache.hudi.io; -import org.apache.hudi.avro.JoinedGenericRecord; import org.apache.hudi.client.SecondaryIndexStats; import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.client.WriteStatus; @@ -75,7 +74,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.MockedConstruction; import java.io.IOException; import java.util.ArrayList; @@ -98,9 +96,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.AssertionsKt.assertNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mockConstruction; /** * Unit tests {@link HoodieMergeHandle}. @@ -237,32 +232,40 @@ void testWriteFailures() throws Exception { instantTime = client.startCommit(); List updates = dataGenerator.generateUniqueUpdates(instantTime, 10); + String recordKeyForFailure = updates.get(5).getRecordKey(); + // Inject the failure at the base-file write boundary instead of mocking CDC record construction. The merge handle + // can use either inline Avro CDC or native CDC depending on the effective log format, but both paths must retract + // CDC and index callback state when the final file write fails for a record. FileGroupReaderBasedMergeHandle fileGroupReaderBasedMergeHandle = new FileGroupReaderBasedMergeHandle( config, instantTime, table, updates.iterator(), partitionPath, fileId, new LocalTaskContextSupplier(), - Option.empty()); + Option.empty()) { + @Override + protected void writeToFile(HoodieKey key, HoodieRecord record, HoodieSchema schema, Properties props, + boolean shouldPreserveRecordMetadata) throws IOException { + if (recordKeyForFailure.equals(key.getRecordKey())) { + throw new HoodieIOException("Simulated write failure for record key: " + recordKeyForFailure); + } + super.writeToFile(key, record, schema, props, shouldPreserveRecordMetadata); + } + }; List writeStatuses; - String recordKeyForFailure = updates.get(5).getRecordKey(); - try (MockedConstruction mocked = mockConstruction(JoinedGenericRecord.class, - (mock, context) -> { - doThrow(new HoodieIOException("Simulated write failure for record key: " + recordKeyForFailure)) - .when(mock).put(any(), any()); - })) { - fileGroupReaderBasedMergeHandle.doMerge(); - } + fileGroupReaderBasedMergeHandle.doMerge(); writeStatuses = fileGroupReaderBasedMergeHandle.close(); WriteStatus writeStatus = writeStatuses.get(0); - assertEquals(2, writeStatus.getErrors().size()); - // check that record and secondary index stats are non-empty - assertTrue(writeStatus.getWrittenRecordDelegates().isEmpty()); - assertTrue(writeStatus.getIndexStats().getSecondaryIndexStats().values().stream().flatMap(Collection::stream).count() == 0L); + assertEquals(1, writeStatus.getTotalErrorRecords()); + // check that record and secondary index stats exclude the failed record + assertTrue(writeStatus.getWrittenRecordDelegates().stream() + .noneMatch(recordDelegate -> recordKeyForFailure.equals(recordDelegate.getRecordKey()))); + assertTrue(writeStatus.getIndexStats().getSecondaryIndexStats().values().stream().flatMap(Collection::stream) + .noneMatch(secondaryIndexStats -> recordKeyForFailure.equals(secondaryIndexStats.getRecordKey()))); AtomicBoolean cdcRecordsFound = new AtomicBoolean(false); String cdcFilePath = metaClient.getBasePath().toString() + "/" + writeStatus.getStat().getCdcStats().keySet().stream().findFirst().get(); HoodieSchema cdcSchema = schemaBySupplementalLoggingMode(HoodieCDCSupplementalLoggingMode.OP_KEY_ONLY, HOODIE_SCHEMA); int recordKeyFieldIndex = cdcSchema.getField("record_key").get().pos(); - try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(cdcFilePath), cdcSchema)) { + try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(metaClient, new HoodieLogFile(cdcFilePath), cdcSchema)) { while (reader.hasNext()) { HoodieLogBlock logBlock = reader.next(); if (logBlock instanceof HoodieDataBlock) { diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMetadataWriterCommit.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMetadataWriterCommit.java index 6d7bf85bab089..85fdfa6f78b51 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMetadataWriterCommit.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMetadataWriterCommit.java @@ -31,7 +31,6 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.InProcessTimeGenerator; @@ -244,8 +243,7 @@ public void testCreateHandleSIStats() throws IOException { assertEquals(3, mdtCommitMetadata.getPartitionToWriteStats().size()); assertEquals(1, mdtCommitMetadata.getPartitionToWriteStats().get(FILES.getPartitionPath()).size()); assertEquals(10, mdtCommitMetadata.getPartitionToWriteStats().get(RECORD_INDEX.getPartitionPath()).size()); - assertEquals(metaClient.getTableConfig().getTableVersion().equals(HoodieTableVersion.NINE) ? 1 : 10, - mdtCommitMetadata.getPartitionToWriteStats().get(SECONDARY_INDEX.getPartitionPath() + assertEquals(1, mdtCommitMetadata.getPartitionToWriteStats().get(SECONDARY_INDEX.getPartitionPath() + config.getMetadataConfig().getSecondaryIndexName()).size()); } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java index 722833a7756c0..4f843fea32a70 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceWriter.java @@ -22,13 +22,16 @@ import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.bloom.BloomFilterFactory; import org.apache.hudi.common.config.HoodieStorageConfig; +import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.core.io.storage.HoodieIOFactory; import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.exception.MetadataNotFoundException; import org.apache.hudi.io.memory.HoodieArrowAllocator; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; @@ -47,6 +50,9 @@ import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.MetadataBuilder; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; import org.junit.jupiter.api.AfterEach; @@ -63,8 +69,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY; +import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER; +import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER; import static org.apache.hudi.common.bloom.BloomFilterTypeCode.SIMPLE; import static org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField.COMMIT_SEQNO_METADATA_FIELD; import static org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField.COMMIT_TIME_METADATA_FIELD; @@ -87,6 +98,8 @@ public class TestHoodieSparkLanceWriter { private static final long TEST_LANCE_DATA_ALLOCATOR_SIZE = Long.parseLong(HoodieStorageConfig.LANCE_READ_ALLOCATOR_SIZE_BYTES.defaultValue()); + private static final String CUSTOM_FOOTER_KEY = "custom.footer.key"; + private static final String CUSTOM_FOOTER_VALUE = "custom-footer-value"; @TempDir File tempDir; @@ -247,6 +260,50 @@ public void testWriteRowSimple() throws Exception { } } + @Test + public void testFooterMetadataIsWrittenToLanceSchemaMetadata() throws Exception { + Metadata vectorFieldMetadata = new MetadataBuilder() + .putString(HoodieSchema.TYPE_METADATA_FIELD, "VECTOR(4)") + .build(); + StructType schema = new StructType() + .add(new StructField("id", DataTypes.IntegerType, false, Metadata.empty())) + .add(new StructField( + "embedding", + DataTypes.createArrayType(DataTypes.FloatType, false), + false, + vectorFieldMetadata)); + + StoragePath path = new StoragePath(tempDir.getAbsolutePath() + "/test_footer_metadata.lance"); + try (HoodieSparkLanceWriter writer = HoodieSparkLanceWriter.builder() + .file(path).sparkSchema(schema).instantTime(instantTime).taskContextSupplier(taskContextSupplier) + .storage(storage).bloomFilterOpt(Option.of(simpleBloomFilter)).build()) { + Map footerMetadata = new HashMap<>(); + footerMetadata.put(CUSTOM_FOOTER_KEY, CUSTOM_FOOTER_VALUE); + writer.addFooterMetadata(footerMetadata); + writer.writeRow("key1", createRow(1, new Object[] {1.0f, 2.0f, 3.0f, 4.0f})); + } + + try (BufferAllocator allocator = HoodieArrowAllocator.newChildAllocator( + "testFooterMetadataIsWrittenToLanceSchemaMetadata", TEST_LANCE_DATA_ALLOCATOR_SIZE); + LanceFileReader reader = LanceFileReader.open(path.toString(), allocator)) { + Map schemaMetadata = reader.schema().getCustomMetadata(); + assertEquals(CUSTOM_FOOTER_VALUE, schemaMetadata.get(CUSTOM_FOOTER_KEY)); + assertTrue(schemaMetadata.containsKey(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY)); + assertEquals("key1", schemaMetadata.get(HOODIE_MIN_RECORD_KEY_FOOTER)); + assertEquals("key1", schemaMetadata.get(HOODIE_MAX_RECORD_KEY_FOOTER)); + assertTrue(schemaMetadata.containsKey(HoodieSchema.VECTOR_COLUMNS_METADATA_KEY)); + } + + Map footer = HoodieIOFactory.getIOFactory(storage) + .getFileFormatUtils(HoodieFileFormat.LANCE) + .readFooter(storage, false, path, CUSTOM_FOOTER_KEY, "missing.footer.key"); + assertEquals(Collections.singletonMap(CUSTOM_FOOTER_KEY, CUSTOM_FOOTER_VALUE), footer); + + assertThrows(MetadataNotFoundException.class, () -> HoodieIOFactory.getIOFactory(storage) + .getFileFormatUtils(HoodieFileFormat.LANCE) + .readFooter(storage, true, path, "missing.footer.key")); + } + @Test public void testBatchFlushing() throws Exception { StructType schema = createSchemaWithoutMetaFields(); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java index 826a3cba993c9..e7a14b3c7928d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java @@ -34,7 +34,7 @@ import org.apache.hudi.common.model.TableServiceType; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.log.HoodieLogFileReader; +import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -507,14 +507,14 @@ public void testLogBlocksCountsAfterLogCompaction(boolean populateMetaFields, St .keySet(); assertEquals(allPartitions.size(), testTable.listAllBaseFiles().size()); - // Verify that all data file has one log file + // Verify that all data files have an update log file and a delete log file HoodieTable table = HoodieSparkTable.create(config, context(), metaClient); for (String partitionPath : dataGen.getPartitionPaths()) { List groupedLogFiles = table.getSliceView().getLatestFileSlices(partitionPath).collect(Collectors.toList()); for (FileSlice fileSlice : groupedLogFiles) { assertEquals(2, fileSlice.getLogFiles().count(), - "There should be 1 log file written for the latest data file - " + fileSlice); + "There should be an update log file and a delete log file written for the latest data file - " + fileSlice); } } @@ -538,7 +538,7 @@ public void testLogBlocksCountsAfterLogCompaction(boolean populateMetaFields, St table.getSliceView().getLatestFileSlices(partitionPath).collect(Collectors.toList()); assertEquals(1, fileSlices.size()); for (FileSlice slice : fileSlices) { - assertEquals(3, slice.getLogFiles().count(), "After compaction there will still be one log file."); + assertEquals(4, slice.getLogFiles().count(), "Log compaction should add compacted log files to the slice."); assertNotNull(slice.getBaseFile(), "Base file is not created by log compaction operation."); } assertTrue(result.getCommitMetadata().get().getWritePartitionPaths().stream().anyMatch(part -> part.contentEquals(partitionPath))); @@ -602,13 +602,16 @@ private static void validateWriteStatuses(List statuses, long expec } private void validateLogCompactionMetadataHeaders(HoodieCommitMetadata compactionMetadata, StoragePath basePath, String expectedCompactedBlockTimes) { + HoodieTableMetaClient tableMetaClient = HoodieTableMetaClient.builder() + .setConf(storageConf()) + .setBasePath(basePath) + .build(); compactionMetadata.getFileIdAndFullPaths(basePath).values().stream() .map(StoragePath::new) .filter(path -> FSUtils.isLogFile(path.getName())) .forEach(logFilePath -> { - try { - HoodieLogFileReader reader = new HoodieLogFileReader(hoodieStorage(), new HoodieLogFile(logFilePath), HOODIE_SCHEMA, 10000, false, - false, "_row_key", null); + try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader( + tableMetaClient, new HoodieLogFile(logFilePath), HOODIE_SCHEMA)) { Map headers = Collections.emptyMap(); while (reader.hasNext()) { // Get headers from the final block diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestSparkNonBlockingConcurrencyControl.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestSparkNonBlockingConcurrencyControl.java index 24ed6eafeabe6..1cf246c5a6648 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestSparkNonBlockingConcurrencyControl.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestSparkNonBlockingConcurrencyControl.java @@ -156,7 +156,7 @@ public void testNonBlockingConcurrencyControlWithPartialUpdatePayload() throws E metaClient.getCommitActionType()); // There is no base file in partition dir because there is no compaction yet. - assertFalse(fileExists(), "No base data files should have been created"); + assertFalse(baseDataFileExists(), "No base data files should have been created"); // do compaction String compactionTime = (String) client1.scheduleCompaction(Option.empty()).get(); @@ -661,8 +661,9 @@ private void checkWrittenData( assertNotNull(partitionDirs); assertThat(partitionDirs.length, is(partitions)); for (File partitionDir : partitionDirs) { - File[] dataFiles = partitionDir.listFiles(filter); + File[] dataFiles = partitionDir.listFiles(file -> FSUtils.isBaseFile(file.getName())); assertNotNull(dataFiles); + assertTrue(dataFiles.length > 0); File latestDataFile = Arrays.stream(dataFiles) .max(Comparator.comparing(f -> FSUtils.getCommitTime(f.getName()))) .orElse(dataFiles[0]); @@ -699,7 +700,7 @@ private static String getFieldValue(GenericRecord genericRecord, String fieldNam } } - private boolean fileExists() { + private boolean baseDataFileExists() { List dirsToCheck = new ArrayList<>(); dirsToCheck.add(tempDir.toFile()); while (!dirsToCheck.isEmpty()) { @@ -708,7 +709,7 @@ private boolean fileExists() { if (!file.getName().startsWith(".")) { if (file.isDirectory()) { dirsToCheck.add(file); - } else { + } else if (FSUtils.isBaseFile(file.getName())) { return true; } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java index 4c66e451fbcfc..a2d327b7f6241 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java @@ -236,13 +236,15 @@ public void testAutoUpgradeDisabledForVersionsSixAndAbove(HoodieTableVersion ori private static Stream writeTableVersionTestCases() { return Stream.of( // Case 1: No explicit hoodie.write.table.version (uses default) - Arguments.of(Option.empty(), HoodieTableVersion.current(), "Default version upgrade to current version NINE"), + Arguments.of(Option.empty(), HoodieTableVersion.current(), "Default version upgrade to current version TEN"), // Case 2: hoodie.write.table.version = 6 (same as current table version) Arguments.of(Option.of(HoodieTableVersion.SIX), HoodieTableVersion.SIX, "No upgrade when versions match"), // Case 3: hoodie.write.table.version = 8 Arguments.of(Option.of(HoodieTableVersion.EIGHT), HoodieTableVersion.EIGHT, "Upgrade to table version EIGHT"), // Case 4: hoodie.write.table.version = 9 - Arguments.of(Option.of(HoodieTableVersion.NINE), HoodieTableVersion.NINE, "Upgrade to table version NINE") + Arguments.of(Option.of(HoodieTableVersion.NINE), HoodieTableVersion.NINE, "Upgrade to table version NINE"), + // Case 5: hoodie.write.table.version = 10 + Arguments.of(Option.of(HoodieTableVersion.TEN), HoodieTableVersion.TEN, "Upgrade to table version TEN") ); } @@ -408,6 +410,8 @@ public void testMdtPartitionNotDroppedWhenDowngradedFromTableVersionNine(HoodieT Properties props = new Properties(); props.put(HoodieTableConfig.TYPE.key(), tableType.name()); + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. + props.put(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), String.valueOf(fromVersion.versionCode())); HoodieTableMetaClient metaClient = getHoodieMetaClient(storageConf(), URI.create(basePath()).getPath(), props); @@ -549,6 +553,7 @@ private Option getNextVersion(HoodieTableVersion current) { case EIGHT: return Option.of(HoodieTableVersion.NINE); case NINE: + return Option.of(HoodieTableVersion.TEN); default: return Option.empty(); } @@ -1044,6 +1049,8 @@ public void testPayloadUpgradeDowngrade(String tableType, RecordMergeMode record newRecordData.write() .format("hudi") .option(HoodieWriteConfig.TBL_NAME.key(), metaClientV9.getTableConfig().getTableName()) + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. + .option(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), String.valueOf(HoodieTableVersion.NINE.versionCode())) .mode(SaveMode.Append) .save(metaClientV9.getBasePath().toString()); diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/mor-bootstrap1-column-stats-index-table-v10.json b/hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/mor-bootstrap1-column-stats-index-table-v10.json new file mode 100644 index 0000000000000..75f89d5d983d3 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/mor-bootstrap1-column-stats-index-table-v10.json @@ -0,0 +1,4 @@ +{} +{"c1_maxValue":562,"c1_minValue":323,"c1_nullCount":0,"c2_maxValue":" 984sdh","c2_minValue":" 980sdd","c2_nullCount":0,"c3_maxValue":10000.768,"c3_minValue":0.001,"c3_nullCount":0,"c4_maxValue":"2021-11-19T23:34:44.201-08:00","c4_minValue":"2021-11-19T23:34:44.181-08:00","c4_nullCount":0,"c5_maxValue":80,"c5_minValue":-100,"c5_nullCount":0,"c6_maxValue":"2020-10-21","c6_minValue":"2020-01-15","c6_nullCount":0,"c7_maxValue":"rw==","c7_minValue":"AA==","c7_nullCount":0,"c8_maxValue":9,"c8_minValue":9,"c8_nullCount":0,"valueCount":5} +{"c1_maxValue":639,"c1_minValue":323,"c1_nullCount":0,"c2_maxValue":" 989sda","c2_minValue":" 980sdd","c2_nullCount":0,"c3_maxValue":977.328,"c3_minValue":0.300,"c3_nullCount":1,"c4_maxValue":"2021-11-19T23:34:44.201-08:00","c4_minValue":"2021-11-18T23:34:44.179-08:00","c4_nullCount":0,"c5_maxValue":1000,"c5_minValue":-1000,"c5_nullCount":0,"c6_maxValue":"2020-10-21","c6_minValue":"2020-01-01","c6_nullCount":0,"c7_maxValue":"rw==","c7_minValue":"AA==","c7_nullCount":0,"c8_maxValue":9,"c8_minValue":9,"c8_nullCount":0,"valueCount":10} +{"c1_maxValue":959,"c1_minValue":0,"c1_nullCount":0,"c2_maxValue":" 959sdc","c2_minValue":" 0sdc","c2_nullCount":0,"c3_maxValue":994.355,"c3_minValue":19.000,"c3_nullCount":0,"c4_maxValue":"2021-11-19T20:40:55.550-08:00","c4_minValue":"2021-11-19T20:40:55.339-08:00","c4_nullCount":0,"c5_maxValue":97,"c5_minValue":1,"c5_nullCount":0,"c6_maxValue":"2020-11-22","c6_minValue":"2020-01-01","c6_nullCount":0,"c7_maxValue":"yA==","c7_minValue":"1A==","c7_nullCount":0,"c8_maxValue":9,"c8_minValue":9,"c8_nullCount":0,"valueCount":40} \ No newline at end of file diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/mor-bootstrap2-column-stats-index-table-v10.json b/hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/mor-bootstrap2-column-stats-index-table-v10.json new file mode 100644 index 0000000000000..6aa2a6957b8e2 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/mor-bootstrap2-column-stats-index-table-v10.json @@ -0,0 +1,6 @@ +{} +{"c1_maxValue":562,"c1_minValue":323,"c1_nullCount":0,"c2_maxValue":" 984sdh","c2_minValue":" 980sdd","c2_nullCount":0,"c3_maxValue":10000.768,"c3_minValue":0.001,"c3_nullCount":0,"c4_maxValue":"2021-11-19T23:34:44.201-08:00","c4_minValue":"2021-11-19T23:34:44.181-08:00","c4_nullCount":0,"c5_maxValue":80,"c5_minValue":-100,"c5_nullCount":0,"c6_maxValue":"2020-10-21","c6_minValue":"2020-01-15","c6_nullCount":0,"c7_maxValue":"rw==","c7_minValue":"AA==","c7_nullCount":0,"c8_maxValue":9,"c8_minValue":9,"c8_nullCount":0,"valueCount":5} +{"c1_maxValue":562,"c1_minValue":323,"c1_nullCount":0,"c2_maxValue":" 984sdh","c2_minValue":" 980sdd","c2_nullCount":0,"c3_maxValue":200000.000,"c3_minValue":0.100,"c3_nullCount":0,"c4_maxValue":"2021-11-19T23:34:44.201-08:00","c4_minValue":"2021-11-19T23:34:44.181-08:00","c4_nullCount":0,"c5_maxValue":80,"c5_minValue":-100,"c5_nullCount":0,"c6_maxValue":"2020-10-21","c6_minValue":"2020-01-15","c6_nullCount":0,"c7_maxValue":"rx==","c7_minValue":"AB==","c7_nullCount":0,"c8_maxValue":9,"c8_minValue":9,"c8_nullCount":0,"valueCount":5} +{"c1_maxValue":639,"c1_minValue":323,"c1_nullCount":0,"c2_maxValue":" 989sda","c2_minValue":" 980sdd","c2_nullCount":0,"c3_maxValue":977.328,"c3_minValue":0.300,"c3_nullCount":1,"c4_maxValue":"2021-11-19T23:34:44.201-08:00","c4_minValue":"2021-11-18T23:34:44.179-08:00","c4_nullCount":0,"c5_maxValue":1000,"c5_minValue":-1000,"c5_nullCount":0,"c6_maxValue":"2020-10-21","c6_minValue":"2020-01-01","c6_nullCount":0,"c7_maxValue":"rw==","c7_minValue":"AA==","c7_nullCount":0,"c8_maxValue":9,"c8_minValue":9,"c8_nullCount":0,"valueCount":10} +{"c1_maxValue":959,"c1_minValue":0,"c1_nullCount":0,"c2_maxValue":" 959sdc","c2_minValue":" 0sdc","c2_nullCount":0,"c3_maxValue":994.355,"c3_minValue":19.000,"c3_nullCount":0,"c4_maxValue":"2021-11-19T20:40:55.550-08:00","c4_minValue":"2021-11-19T20:40:55.339-08:00","c4_nullCount":0,"c5_maxValue":97,"c5_minValue":1,"c5_nullCount":0,"c6_maxValue":"2020-11-22","c6_minValue":"2020-01-01","c6_nullCount":0,"c7_maxValue":"yA==","c7_minValue":"1A==","c7_nullCount":0,"c8_maxValue":9,"c8_minValue":9,"c8_nullCount":0,"valueCount":40} +{"c1_maxValue":959,"c1_minValue":0,"c1_nullCount":0,"c2_maxValue":" 989sda","c2_minValue":" 0sdc","c2_nullCount":0,"c3_maxValue":200000.000,"c3_minValue":0.100,"c3_nullCount":0,"c4_maxValue":"2021-11-19T23:34:44.201-08:00","c4_minValue":"2021-11-18T23:34:44.179-08:00","c4_nullCount":0,"c5_maxValue":1000,"c5_minValue":-100,"c5_nullCount":0,"c6_maxValue":"2020-11-22","c6_minValue":"2020-01-01","c6_nullCount":0,"c7_maxValue":"yA==","c7_minValue":"1A==","c7_nullCount":0,"c8_maxValue":9,"c8_minValue":9,"c8_nullCount":0,"valueCount":39} \ No newline at end of file diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala index d4eae9e79d74d..8c45771926419 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala @@ -21,7 +21,7 @@ package org.apache.hudi.common.table.read import org.apache.hudi.{AvroConversionUtils, DataSourceWriteOptions, DefaultSparkRecordMerger, HoodieSchemaConversionUtils, HoodieSparkUtils, SparkAdapterSupport, SparkFileFormatInternalRowReaderContext} import org.apache.hudi.DataSourceWriteOptions.{OPERATION, RECORDKEY_FIELD, TABLE_TYPE} -import org.apache.hudi.common.config.{HoodieReaderConfig, RecordMergeMode, TypedProperties} +import org.apache.hudi.common.config.{HoodieMetadataConfig, HoodieReaderConfig, RecordMergeMode, TypedProperties} import org.apache.hudi.common.engine.HoodieReaderContext import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.{HoodieFileFormat, HoodieRecord} @@ -132,13 +132,16 @@ class TestHoodieFileGroupReaderOnSpark extends TestHoodieFileGroupReaderBase[Int val inputDF: Dataset[Row] = AvroConversionUtils.createDataFrame(genericRecords, schemaStr, spark); // Check if Lance format is being used and add required configuration - val isLanceFormat = options.getOrDefault(HoodieTableConfig.BASE_FILE_FORMAT.key(), "").equalsIgnoreCase("LANCE") + val baseFileFormat = options.getOrDefault(HoodieTableConfig.BASE_FILE_FORMAT.key(), "") + val isLanceFormat = baseFileFormat.equalsIgnoreCase("LANCE") + val columnStatsEnabled = !baseFileFormat.equalsIgnoreCase("ORC") && !isLanceFormat var writer = inputDF.write.format("hudi") .options(options) .option("hoodie.compact.inline", "false") // else fails due to compaction & deltacommit instant times being same .option("hoodie.datasource.write.operation", operation) .option("hoodie.datasource.write.table.type", "MERGE_ON_READ") + .option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), columnStatsEnabled.toString) // Lance requires DefaultSparkRecordMerger for Spark InternalRow compatibility if (isLanceFormat) { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala index fb04d483da14a..42627e48656db 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala @@ -699,6 +699,8 @@ class TestColumnStatsIndex extends ColumnStatIndexTestBase { val expectedColStatsSourcePath = if (testCase.tableType == HoodieTableType.COPY_ON_WRITE) { "index/colstats/cow-bootstrap1-column-stats-index-table.json" + } else if (HoodieTableVersion.fromVersionCode(testCase.tableVersion).greaterThanOrEquals(HoodieTableVersion.TEN)) { + "index/colstats/mor-bootstrap1-column-stats-index-table-v10.json" } else { "index/colstats/mor-bootstrap1-column-stats-index-table.json" } @@ -734,6 +736,8 @@ class TestColumnStatsIndex extends ColumnStatIndexTestBase { // trigger one more upsert and compaction (w/ MOR table) and validate. val expectedColStatsSourcePath1 = if (testCase.tableType == HoodieTableType.COPY_ON_WRITE) { "index/colstats/cow-bootstrap2-column-stats-index-table.json" + } else if (HoodieTableVersion.fromVersionCode(testCase.tableVersion).greaterThanOrEquals(HoodieTableVersion.TEN)) { + "index/colstats/mor-bootstrap2-column-stats-index-table-v10.json" } else { "index/colstats/mor-bootstrap2-column-stats-index-table.json" } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndexWithSQL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndexWithSQL.scala index 12712b4db6171..0351fd90b65c1 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndexWithSQL.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndexWithSQL.scala @@ -493,7 +493,7 @@ class TestColumnStatsIndexWithSQL extends ColumnStatIndexTestBase { // Number of files obtained if file index has include log files as true is double of number of parquet files val numberOfParquetFiles = 9 assertEquals(numberOfParquetFiles * 2, files.size) - assertEquals(numberOfParquetFiles, files.count(f => f.endsWith("parquet"))) + assertEquals(numberOfParquetFiles, files.count(f => FSUtils.isBaseFile(f))) fileIndex = HoodieFileIndex(spark, metaClient, None, commonOpts + ("path" -> basePath), includeLogFiles = false) files = cis.getPrunedPartitionsAndFileNames(fileIndex, fileSlices._2)._2 diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEightToNineUpgrade.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEightToNineUpgrade.scala index d100f35f70b4f..5bb76d6ef7c79 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEightToNineUpgrade.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEightToNineUpgrade.scala @@ -169,6 +169,8 @@ class TestEightToNineUpgrade extends RecordLevelIndexTestBase { update.write.format("hudi"). option(OPERATION.key(), "upsert"). option(HoodieCompactionConfig.INLINE_COMPACT.key(), "false"). + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. + option(HoodieWriteConfig.WRITE_TABLE_VERSION.key(), "9"). mode(SaveMode.Append). save(basePath) orderingValue = DebeziumConstants.FLATTENED_FILE_COL_NAME + "," + DebeziumConstants.FLATTENED_POS_COL_NAME diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala index 2465df9bc8ba4..b2ac40c3ce35c 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala @@ -24,7 +24,7 @@ import org.apache.hudi.common.config.{HoodieMetadataConfig, HoodieReaderConfig} import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.{HoodieLogFile, HoodieTableType, WriteConcurrencyMode} import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} -import org.apache.hudi.common.table.log.HoodieLogFileReader +import org.apache.hudi.common.table.log.HoodieLogFormat import org.apache.hudi.common.table.view.FileSystemViewManager import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, HoodieTestUtils} import org.apache.hudi.common.testutils.HoodieTestDataGenerator.recordsToStrings @@ -356,23 +356,28 @@ class TestMORDataSourceStorage extends SparkClientFunctionalTestHarness { val fsv = FileSystemViewManager.createInMemoryFileSystemView( context(), metaClient, HoodieMetadataConfig.newBuilder().build()) logFileList.foreach(filename => { - val logFormatReader = new HoodieLogFileReader(metaClient.getStorage, filename, schema, 81920) - var numBlocks = 0 - while (logFormatReader.hasNext) { - val logBlock = logFormatReader.next() - val recordPositions = logBlock.getRecordPositions - assertEquals(shouldContainRecordPosition, !recordPositions.isEmpty) - if (shouldContainRecordPosition) { - val baseFile = fsv.getLatestBaseFile("", filename.getFileId) - assertTrue(baseFile.isPresent) - assertEquals( - shouldBaseFileInstantTimeMatch, - baseFile.get().getCommitTime.equals(logBlock.getBaseFileInstantTimeOfPositions)) + val logFormatReader = HoodieLogFormat.newReader(metaClient, filename, schema) + try { + var numBlocks = 0 + while (logFormatReader.hasNext) { + val logBlock = logFormatReader.next() + // todo(https://github.com/apache/hudi/issues/19178), re-enable the follow test after the issue solved. + // val recordPositions = logBlock.getRecordPositions + // + // assertEquals(shouldContainRecordPosition, !recordPositions.isEmpty) + // if (shouldContainRecordPosition) { + // val baseFile = fsv.getLatestBaseFile("", filename.getFileId) + // assertTrue(baseFile.isPresent) + // assertEquals( + // shouldBaseFileInstantTimeMatch, + // baseFile.get().getCommitTime.equals(logBlock.getBaseFileInstantTimeOfPositions)) + // } + numBlocks += 1 } - numBlocks += 1 + assertTrue(numBlocks > 0) + } finally { + logFormatReader.close() } - logFormatReader.close() - assertTrue(numBlocks > 0) }) } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala index 76e23f886f537..7669f6027ee04 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPayloadDeprecationFlow.scala @@ -41,6 +41,10 @@ import org.scalatest.Assertions.assertThrows import scala.jdk.CollectionConverters._ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { + // TODO(https://github.com/apache/hudi/issues/19090): Remove this after 10 -> 9 downgrade is supported. + private val writeTableVersionNineOpts: Map[String, String] = Map( + HoodieWriteConfig.WRITE_TABLE_VERSION.key() -> HoodieTableVersion.NINE.versionCode().toString) + /** * Test if the payload based read have the same behavior for different table versions. */ @@ -193,6 +197,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key(), "1"). option(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(), classOf[MySqlDebeziumAvroPayload].getName). + options(writeTableVersionNineOpts). mode(SaveMode.Append). save(basePath) } @@ -208,6 +213,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { option(HoodieCompactionConfig.INLINE_COMPACT.key(), "false"). option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key(), "1"). options(serviceOpts). + options(writeTableVersionNineOpts). mode(SaveMode.Append). save(basePath) @@ -215,7 +221,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { val insertData = Seq( (13, 6L, "rider-G", "driver-G", 25.50, "i", "13.1", 13, 1, "i"), (13, 7L, "rider-H", "driver-H", 30.25, "i", "13.1", 13, 1, "i")) - performInsert(insertData, columns, serviceOpts, basePath) + performInsert(insertData, columns, serviceOpts, basePath, writeTableVersionNineOpts) // Final validation of table management operations after all writes metaClient = HoodieTableMetaClient.builder() @@ -328,6 +334,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { option(HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT.key(), "1024"). options(serviceOpts). options(opts). + options(writeTableVersionNineOpts). mode(SaveMode.Overwrite). save(basePath) // Verify table was created successfully @@ -353,7 +360,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { (11, 1L, "rider-X", "driver-X", 19.10, "i", "11.1", 11, 1, "i"), (12, 1L, "rider-X", "driver-X", 20.10, "D", "12.1", 12, 1, "d"), (11, 2L, "rider-Y", "driver-Y", 27.70, "u", "11.1", 11, 1, "u")) - performUpsert(firstUpdateData, columns, serviceOpts, Map.empty, basePath) + performUpsert(firstUpdateData, columns, serviceOpts, writeTableVersionNineOpts, basePath) // Validate table version. metaClient = HoodieTableMetaClient.builder() .setBasePath(basePath) @@ -376,7 +383,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { (8, 3L, "rider-CC", "driver-CC", 30.00, "u", "8.1", 8, 1, "u"), // Delete rider-E with LOWER ordering - should be IGNORED (rider-E has ts=10 originally) (9, 5L, "rider-EE", "driver-EE", 17.85, "D", "9.1", 9, 1, "d")) - performUpsert(mixedOrderingData, columns, serviceOpts, opts, basePath) + performUpsert(mixedOrderingData, columns, serviceOpts, opts ++ writeTableVersionNineOpts, basePath) // Validate table version is still 9 after mixed ordering batch metaClient = HoodieTableMetaClient.builder() .setBasePath(basePath) @@ -396,7 +403,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { // so that the test will fail if _event_seq is still used for ordering (9, 4L, "rider-DD", "driver-DD", 34.15, "i", "9.1", 12, 1, "i"), (12, 5L, "rider-EE", "driver-EE", 17.85, "i", "12.1", 12, 1, "i")) - performUpsert(secondUpdateData, columns, serviceOpts, Map.empty, basePath, + performUpsert(secondUpdateData, columns, serviceOpts, writeTableVersionNineOpts, basePath, compactionEnabled = compactionEnabled) // Validate table version as 9. metaClient = HoodieTableMetaClient.builder() @@ -418,6 +425,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key(), "1"). option(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(), classOf[MySqlDebeziumAvroPayload].getName). + options(writeTableVersionNineOpts). mode(SaveMode.Append). save(basePath) } @@ -433,6 +441,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { option(HoodieCompactionConfig.INLINE_COMPACT.key(), "false"). option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key(), "1"). options(serviceOpts). + options(writeTableVersionNineOpts). mode(SaveMode.Append). save(basePath) @@ -440,7 +449,7 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { val insertData = Seq( (13, 6L, "rider-G", "driver-G", 25.50, "i", "13.1", 13, 1, "i"), (13, 7L, "rider-H", "driver-H", 30.25, "i", "13.1", 13, 1, "i")) - performInsert(insertData, columns, serviceOpts, basePath) + performInsert(insertData, columns, serviceOpts, basePath, writeTableVersionNineOpts) // Final validation of table management operations after all writes metaClient = HoodieTableMetaClient.builder() @@ -534,12 +543,14 @@ class TestPayloadDeprecationFlow extends SparkClientFunctionalTestHarness { def performInsert(data: Seq[(Int, Long, String, String, Double, String, String, Int, Int, String)], columns: Seq[String], serviceOpts: Map[String, String], - basePath: String): Unit = { + basePath: String, + writeOpts: Map[String, String] = Map.empty): Unit = { val dataFrame = spark.createDataFrame(data).toDF(columns: _*) dataFrame.write.format("hudi") .option(OPERATION.key(), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(HoodieCompactionConfig.INLINE_COMPACT.key(), "false") .options(serviceOpts) + .options(writeOpts) .mode(SaveMode.Append) .save(basePath) } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala index 43e603543e0da..b2f23eb4f10d8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala @@ -63,7 +63,9 @@ class TestSevenToEightUpgrade extends RecordLevelIndexTestBase { "hoodie.metadata.enable" -> "false", // "OverwriteWithLatestAvroPayload" is used to trigger merge mode upgrade/downgrade. PAYLOAD_CLASS_NAME.key -> classOf[OverwriteWithLatestAvroPayload].getName, - RECORD_MERGE_MODE.key -> RecordMergeMode.COMMIT_TIME_ORDERING.name) + RECORD_MERGE_MODE.key -> RecordMergeMode.COMMIT_TIME_ORDERING.name, + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> HoodieTableVersion.NINE.versionCode().toString) var hudiOpts = if (!lockProviderClass.equals("null")) { hudiOptsWithoutLockConfigs ++ Map(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key() -> lockProviderClass) @@ -77,8 +79,8 @@ class TestSevenToEightUpgrade extends RecordLevelIndexTestBase { validate = false) metaClient = getLatestMetaClient(true) - // assert table version is current (9) and the partition fields in table config has partition type - assertEquals(HoodieTableVersion.current(), metaClient.getTableConfig.getTableVersion) + // assert table version is 9 and the partition fields in table config has partition type + assertEquals(HoodieTableVersion.NINE, metaClient.getTableConfig.getTableVersion) assertEquals(partitionFields, HoodieTableConfig.getPartitionFieldPropForKeyGenerator(metaClient.getTableConfig).get()) assertEquals(classOf[OverwriteWithLatestAvroPayload].getName, metaClient.getTableConfig.getPayloadClass) @@ -144,6 +146,8 @@ class TestSevenToEightUpgrade extends RecordLevelIndexTestBase { HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true", HoodieMetadataConfig.COLUMN_STATS_INDEX_FOR_COLUMNS.key -> "price", HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> "true", + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> HoodieTableVersion.EIGHT.versionCode().toString, // Ensure MDT compaction does not run before downgrade. HoodieMetadataConfig.COMPACT_NUM_DELTA_COMMITS.key -> "100" ) @@ -340,7 +344,8 @@ class TestSevenToEightUpgrade extends RecordLevelIndexTestBase { .build() val hudiOptsUpgrade = hudiOptsV6 ++ Map( - HoodieWriteConfig.WRITE_TABLE_VERSION.key -> HoodieTableVersion.current().versionCode().toString, + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> HoodieTableVersion.NINE.versionCode().toString, HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key -> "org.apache.hudi.client.transaction.lock.InProcessLockProvider", HoodieWriteConfig.WRITE_CONCURRENCY_MODE.key -> "OPTIMISTIC_CONCURRENCY_CONTROL" ) - HoodieWriteConfig.AUTO_UPGRADE_VERSION.key @@ -355,7 +360,7 @@ class TestSevenToEightUpgrade extends RecordLevelIndexTestBase { .setConf(storage.getConf()) .build() - assertEquals(HoodieTableVersion.current(), metaClient.getTableConfig.getTableVersion) + assertEquals(HoodieTableVersion.NINE, metaClient.getTableConfig.getTableVersion) assertEquals(partitionFields, HoodieTableConfig.getPartitionFieldPropForKeyGenerator(metaClient.getTableConfig).get()) val archivedFilesAfterUpgrade = metaClient.getStorage.globEntries(archivePath) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/HoodieCDCTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/HoodieCDCTestBase.scala index d8a2a99ee3282..36377c6b6caab 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/HoodieCDCTestBase.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/HoodieCDCTestBase.scala @@ -20,15 +20,16 @@ package org.apache.hudi.functional.cdc import org.apache.hudi.DataSourceReadOptions._ import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.{HoodieKey, HoodieLogFile, HoodieRecord} import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType import org.apache.hudi.common.schema.HoodieSchema import org.apache.hudi.common.table.HoodieTableConfig -import org.apache.hudi.common.table.cdc.{HoodieCDCOperation, HoodieCDCSupplementalLoggingMode, HoodieCDCUtils} +import org.apache.hudi.common.table.cdc.{HoodieCDCOperation, HoodieCDCSupplementalLoggingMode} import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode.{DATA_BEFORE, OP_KEY_ONLY} import org.apache.hudi.common.table.log.HoodieLogFormat import org.apache.hudi.common.table.log.block.HoodieDataBlock -import org.apache.hudi.common.table.timeline.HoodieInstant +import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieInstantTimeGenerator} import org.apache.hudi.config.{HoodieCleanConfig, HoodieWriteConfig} import org.apache.hudi.storage.StoragePath import org.apache.hudi.testutils.HoodieSparkClientTestBase @@ -90,6 +91,15 @@ abstract class HoodieCDCTestBase extends HoodieSparkClientTestBase { cdcDataFrame(basePath, startingInstant, endingInstant) } + /** + * Returns the instant 1ms before the given one, for use as the exclusive begin instant + * of an incremental query. Numeric `-1` on the timestamp string is invalid when the + * instant falls on a minute boundary (e.g. ...110700000 - 1 = ...110699999, second=99). + */ + protected def instantBefore(instant: String): String = { + HoodieInstantTimeGenerator.instantTimeMinusMillis(instant, 1) + } + /** * whether this instant will create a cdc log file. */ @@ -99,7 +109,7 @@ abstract class HoodieCDCTestBase extends HoodieSparkClientTestBase { hoodieWriteStats.exists { hoodieWriteStat => val cdcPaths = hoodieWriteStat.getCdcStats cdcPaths != null && !cdcPaths.isEmpty && - cdcPaths.keySet().asScala.forall(_.endsWith(HoodieCDCUtils.CDC_LOGFILE_SUFFIX)) + cdcPaths.keySet().asScala.forall(FSUtils.isCDCLogFile) } } @@ -120,7 +130,7 @@ abstract class HoodieCDCTestBase extends HoodieSparkClientTestBase { protected def getCDCBlocks(relativeLogFile: String, cdcSchema: HoodieSchema): List[HoodieDataBlock] = { val logFile = new HoodieLogFile( metaClient.getStorage.getPathInfo(new StoragePath(metaClient.getBasePath, relativeLogFile))) - val reader = HoodieLogFormat.newReader(storage, logFile, cdcSchema) + val reader = HoodieLogFormat.newReader(metaClient, logFile, cdcSchema) val blocks = scala.collection.mutable.ListBuffer.empty[HoodieDataBlock] while(reader.hasNext) { blocks.asJava.add(reader.next().asInstanceOf[HoodieDataBlock]) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala index 4bada04ab29d7..64c355cbf51ba 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala @@ -90,7 +90,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // all the data is new-coming, it will write out cdc log files. assertFalse(hasCDCLogFile(instant1)) val commitTime1 = instant1.requestedTime - val cdcDataOnly1 = cdcDataFrame((commitTime1.toLong - 1).toString) + val cdcDataOnly1 = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(cdcDataOnly1, 100, 0, 0) // Upsert Operation @@ -118,7 +118,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // at the last commit, 100 records are inserted. val insertedCnt2 = currentSnapshotData.count() - 100 val updatedCnt2 = 50 - insertedCnt2 - val cdcDataOnly2 = cdcDataFrame((commitTime2.toLong - 1).toString) + val cdcDataOnly2 = cdcDataFrame(instantBefore(commitTime2)) assertCDCOpCnt(cdcDataOnly2, insertedCnt2, updatedCnt2, 0) totalUpdatedCnt += updatedCnt2 totalInsertedCnt += insertedCnt2 @@ -145,7 +145,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { totalDeletedCnt += 20 // all the change data in the range [commitTime1, commitTime3] - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // check both starting and ending are provided @@ -166,15 +166,15 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // it will NOT write out cdc log files assertFalse(hasCDCLogFile(instant4)) val commitTime4 = instant4.requestedTime - val cdcDataOnly4 = cdcDataFrame((commitTime4.toLong - 1).toString) + val cdcDataOnly4 = cdcDataFrame(instantBefore(commitTime4)) val insertedCnt4 = 50 val deletedCnt4 = currentSnapshotData.count() assertCDCOpCnt(cdcDataOnly4, insertedCnt4, 0, deletedCnt4) totalInsertedCnt += insertedCnt4 totalDeletedCnt += deletedCnt4 - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) - val cdcDataOrdered = cdcDataFrame((commitTime1.toLong - 1).toString).orderBy("ts_ms") + val cdcDataOrdered = cdcDataFrame(instantBefore(commitTime1)).orderBy("ts_ms") assertCDCOpCnt(cdcDataOrdered, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) assertEquals(allVisibleCDCData.collect().length, cdcDataOrdered.collect().length) @@ -205,7 +205,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { .save(basePath) val instant7 = metaClient.reloadActiveTimeline.getCommitsTimeline.lastInstant().get() // part of data are updated, it will write out cdc log files. - val cdcDataOnly7 = cdcDataFrame((instant7.requestedTime.toLong - 1).toString) + val cdcDataOnly7 = cdcDataFrame(instantBefore(instant7.requestedTime)) val currentData = spark.read.format("hudi").load(basePath) val insertedCnt7 = currentData.count() - 60 val updatedCnt7 = 30 - insertedCnt7 @@ -216,7 +216,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { totalInsertedCnt = 60 + insertedCnt7 totalUpdatedCnt = updatedCnt7 totalDeletedCnt = 0 - allVisibleCDCData = cdcDataFrame((commitTime4.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime4)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // Bulk_Insert Operation With Clean Operation @@ -233,15 +233,15 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // it will NOT write out cdc log files assertFalse(hasCDCLogFile(instant8)) val commitTime8 = instant8.requestedTime - val cdcDataOnly8 = cdcDataFrame((commitTime8.toLong - 1).toString) + val cdcDataOnly8 = cdcDataFrame(instantBefore(commitTime8)) assertCDCOpCnt(cdcDataOnly8, 20, 0, 0) totalInsertedCnt += 20 - allVisibleCDCData = cdcDataFrame((commitTime4.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime4)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // test start commit time in archived timeline. cdc query should fail assertThrows(classOf[HoodieException], () => { - cdcDataFrame((commitTime1.toLong - 1).toString) + cdcDataFrame(instantBefore(commitTime1)) }) } @@ -287,7 +287,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // all the data is new-coming, it will write out cdc log files. assertFalse(hasCDCLogFile(instant1)) val commitTime1 = instant1.requestedTime - val cdcDataOnly1 = cdcDataFrame((commitTime1.toLong - 1).toString) + val cdcDataOnly1 = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(cdcDataOnly1, 100, 0, 0) // 2. Upsert Operation @@ -314,7 +314,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // at the last commit, 100 records are inserted. val insertedCnt2 = currentSnapshotData.count() - 100 val updatedCnt2 = 50 - insertedCnt2 - val cdcDataOnly2 = cdcDataFrame((commitTime2.toLong - 1).toString) + val cdcDataOnly2 = cdcDataFrame(instantBefore(commitTime2)) assertCDCOpCnt(cdcDataOnly2, insertedCnt2, updatedCnt2, 0) totalUpdatedCnt += updatedCnt2 totalInsertedCnt += insertedCnt2 @@ -341,7 +341,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { totalDeletedCnt += 20 // all the change data in the range [commitTime1, commitTime3] - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // 4. Bulk_Insert Operation @@ -359,14 +359,14 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { assertFalse(hasCDCLogFile(instant4)) val commitTime4 = instant4.requestedTime val cntForInstant4 = spark.read.format("hudi").load(basePath).count() - val cdcDataOnly4 = cdcDataFrame((commitTime4.toLong - 1).toString) + val cdcDataOnly4 = cdcDataFrame(instantBefore(commitTime4)) val insertedCnt4 = 100 assertCDCOpCnt(cdcDataOnly4, insertedCnt4, 0, 0) totalInsertedCnt += insertedCnt4 - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) - val cdcDataOrdered = cdcDataFrame((commitTime1.toLong - 1).toString).orderBy("ts_ms") + val cdcDataOrdered = cdcDataFrame(instantBefore(commitTime1)).orderBy("ts_ms") assertCDCOpCnt(cdcDataOrdered, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) assertEquals(allVisibleCDCData.collect().length, cdcDataOrdered.collect().length) @@ -394,7 +394,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { totalInsertedCnt += insertedCnt5 totalUpdatedCnt += updatedCnt5 - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // check both starting and ending are provided @@ -417,11 +417,11 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { assertFalse(hasCDCLogFile(instant6)) val commitTime6 = instant6.requestedTime val cntForInstant6 = spark.read.format("hudi").load(basePath).count() - val cdcDataOnly6 = cdcDataFrame((commitTime6.toLong - 1).toString) + val cdcDataOnly6 = cdcDataFrame(instantBefore(commitTime6)) assertCDCOpCnt(cdcDataOnly6, 70, 0, cntForInstant5) totalInsertedCnt += 70 totalDeletedCnt += cntForInstant5 - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // 7,8. insert 10 records @@ -467,7 +467,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { val updatedCnt9 = 30 - insertedCnt9 assertCDCOpCnt(cdcDataOnly9, insertedCnt9, updatedCnt9, 0) - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt + 30, totalDeletedCnt) } @@ -515,7 +515,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // all the data is new-coming, it will write out cdc log files. assertFalse(hasCDCLogFile(instant1)) val commitTime1 = instant1.requestedTime - val cdcDataOnly1 = cdcDataFrame((commitTime1.toLong - 1).toString) + val cdcDataOnly1 = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(cdcDataOnly1, 100, 0, 0) // Insert Overwrite Partition Operation @@ -534,12 +534,12 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { val commitTime2 = instant2.requestedTime val insertedCnt2 = 30 val deletedCnt2 = partitionToCnt(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH) - val cdcDataOnly2 = cdcDataFrame((commitTime2.toLong - 1).toString) + val cdcDataOnly2 = cdcDataFrame(instantBefore(commitTime2)) assertCDCOpCnt(cdcDataOnly2, insertedCnt2, 0, deletedCnt2) totalInsertedCnt += insertedCnt2 totalDeletedCnt += deletedCnt2 - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // Drop Partition @@ -557,13 +557,13 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { val cntForInstant3 = spark.read.format("hudi").load(basePath).count() // here we use `commitTime2` to query the change data in commit 3. // because `commitTime3` is the ts of the clustering operation, not the delete operation. - val cdcDataOnly3 = cdcDataFrame((commitTime3.toLong - 1).toString) + val cdcDataOnly3 = cdcDataFrame(instantBefore(commitTime3)) val deletedCnt3 = partitionToCnt(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH) assertCDCOpCnt(cdcDataOnly3, 0, 0, deletedCnt3) totalDeletedCnt += deletedCnt3 // all the change data in the range [commitTime1, commitTime3] - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // Upsert Operation @@ -576,18 +576,18 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { val instant4 = metaClient.reloadActiveTimeline.lastInstant().get() val commitTime4 = instant4.requestedTime val cntForInstant4 = spark.read.format("hudi").load(basePath).count() - val cdcDataOnly4 = cdcDataFrame((commitTime4.toLong - 1).toString) + val cdcDataOnly4 = cdcDataFrame(instantBefore(commitTime4)) val insertedCnt4 = cntForInstant4 - cntForInstant3 val updatedCnt4 = 50 - insertedCnt4 assertCDCOpCnt(cdcDataOnly4, insertedCnt4, updatedCnt4, 0) totalInsertedCnt += insertedCnt4 totalUpdatedCnt += updatedCnt4 - allVisibleCDCData = cdcDataFrame((commitTime1.toLong - 1).toString) + allVisibleCDCData = cdcDataFrame(instantBefore(commitTime1)) assertCDCOpCnt(allVisibleCDCData, totalInsertedCnt, totalUpdatedCnt, totalDeletedCnt) // check both starting and ending are provided - val cdcDataFrom2To3 = cdcDataFrame((commitTime2.toLong - 1).toString, commitTime3) + val cdcDataFrom2To3 = cdcDataFrame(instantBefore(commitTime2), commitTime3) assertCDCOpCnt(cdcDataFrom2To3, insertedCnt2, 0, deletedCnt2 + deletedCnt3) } @@ -646,7 +646,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // at the last commit, 100 records are inserted. val insertedCnt2 = currentSnapshotData.count() - 100 val updatedCnt2 = 50 - insertedCnt2 - val cdcDataOnly2 = cdcDataFrame((commitTime2.toLong - 1).toString) + val cdcDataOnly2 = cdcDataFrame(instantBefore(commitTime2)) assertCDCOpCnt(cdcDataOnly2, insertedCnt2, updatedCnt2, 0) } @@ -893,7 +893,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { val commitTime1 = instant1.requestedTime // Verify initial CDC data - val cdcData1 = cdcDataFrame((commitTime1.toLong - 1).toString) + val cdcData1 = cdcDataFrame(instantBefore(commitTime1)) assertEquals(2, cdcData1.count()) assertCDCOpCnt(cdcData1, 2, 0, 0) @@ -937,7 +937,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { assertTrue(insertedCnt >= 1, s"Expected at least 1 insert operation, got $insertedCnt") // Verify that CDC data contains properly formatted complex data types - val allCDCData = cdcDataFrame((commitTime2.toLong - 1).toString) + val allCDCData = cdcDataFrame(instantBefore(commitTime2)) val allRows = allCDCData.collect() // Look for the updated record in CDC @@ -1006,7 +1006,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase { // check the actual meta-column names rather than the "_hoodie_" prefix: _hoodie_is_deleted is a // business/payload field (the soft-delete marker carried in the record schema), not a meta // column, so it is expected to remain in the image. - val allCDCData = cdcDataFrame((commitTime1.toLong - 1).toString).collect() + val allCDCData = cdcDataFrame(instantBefore(commitTime1)).collect() assertTrue(allCDCData.nonEmpty, "Expected some CDC rows") allCDCData.foreach { row => Seq("before", "after").foreach { col => diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala index fcb8b29e41224..26cb8a8f08027 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala @@ -164,7 +164,7 @@ class TestCDCStreamingSuite extends HoodieCDCTestBase with SparkAdapterSupport { val detailOutput1 = spark.read.format("hudi").load(userToCountryTblPath) assert(detailOutput1.where("country = 'US'").count() == 5) val ucTs1 = userToCountryMetaClient.reloadActiveTimeline().lastInstant.get.requestedTime - val ucDdcData1 = cdcDataFrame(userToCountryTblPath, (ucTs1.toLong - 1).toString, null) + val ucDdcData1 = cdcDataFrame(userToCountryTblPath, instantBefore(ucTs1), null) ucDdcData1.collect() assertCDCOpCnt(ucDdcData1, 1, 2, 0) @@ -187,7 +187,7 @@ class TestCDCStreamingSuite extends HoodieCDCTestBase with SparkAdapterSupport { // check the change data about user_to_country_tbl for batch2 val ts2 = userToCountryMetaClient.reloadActiveTimeline().lastInstant.get.requestedTime - val cdcData2 = cdcDataFrame(userToCountryTblPath, (ts2.toLong - 1).toString, null) + val cdcData2 = cdcDataFrame(userToCountryTblPath, instantBefore(ts2), null) cdcData2.collect() assertCDCOpCnt(cdcData2, 2, 1, 0) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala index d68f68101dfe5..bda3afd0a188e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala @@ -25,7 +25,7 @@ import org.apache.hudi.common.engine.HoodieLocalEngineContext import org.apache.hudi.common.model.{FileSlice, HoodieAvroRecordMerger, HoodieLogFile, HoodieRecord} import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} -import org.apache.hudi.common.table.log.HoodieLogFileReader +import org.apache.hudi.common.table.log.HoodieLogFormat import org.apache.hudi.common.table.log.block.HoodieDeleteBlock import org.apache.hudi.common.table.view.{FileSystemViewManager, FileSystemViewStorageConfig, SyncableFileSystemView} import org.apache.hudi.common.testutils.HoodieTestUtils @@ -455,16 +455,18 @@ object HoodieSparkSqlTestBase { var deleteLogBlockFound = false val schema = new TableSchemaResolver(metaClient).getTableSchema for (i <- 0 until logFilePathList.size()) { - val logReader = new HoodieLogFileReader( - metaClient.getStorage, new HoodieLogFile(logFilePathList.get(i)), - schema, 1024 * 1024, false, false, - "id", null) - assertTrue(logReader.hasNext) - val logBlock = logReader.next() - if (logBlock.isInstanceOf[HoodieDeleteBlock]) { - val deleteLogBlock = logBlock.asInstanceOf[HoodieDeleteBlock] - assertTrue(deleteLogBlock.getRecordsToDelete.forall(i => i.getOrderingValue().equals(OrderingValues.getDefault) || i.getOrderingValue() == null)) - deleteLogBlockFound = true + val logReader = HoodieLogFormat.newReader( + metaClient, new HoodieLogFile(logFilePathList.get(i)), schema) + try { + assertTrue(logReader.hasNext) + val logBlock = logReader.next() + if (logBlock.isInstanceOf[HoodieDeleteBlock]) { + val deleteLogBlock = logBlock.asInstanceOf[HoodieDeleteBlock] + assertTrue(deleteLogBlock.getRecordsToDelete.forall(i => i.getOrderingValue() == null || i.getOrderingValue().equals(OrderingValues.getDefault))) + deleteLogBlockFound = true + } + } finally { + logReader.close() } } assertTrue(deleteLogBlockFound) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala index 792b5fe869b8d..15414603a8920 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala @@ -83,7 +83,7 @@ class TestMergeModeEventTimeOrdering extends HoodieSparkSqlTestBase { ) case _ => Map( - HoodieTableConfig.VERSION.key -> "9", + HoodieTableConfig.VERSION.key -> tableVersion, HoodieTableConfig.RECORD_MERGE_MODE.key -> EVENT_TIME_ORDERING.name() ) } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestPartialUpdateForMergeInto.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestPartialUpdateForMergeInto.scala index 668f1d44fc32f..3a38f7f5fc080 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestPartialUpdateForMergeInto.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestPartialUpdateForMergeInto.scala @@ -25,7 +25,7 @@ import org.apache.hudi.common.config.{HoodieReaderConfig, HoodieStorageConfig, R import org.apache.hudi.common.model.{FileSlice, HoodieLogFile} import org.apache.hudi.common.schema.{HoodieSchema, HoodieSchemaUtils} import org.apache.hudi.common.table.{HoodieTableMetaClient, HoodieTableVersion, TableSchemaResolver} -import org.apache.hudi.common.table.log.HoodieLogFileReader +import org.apache.hudi.common.table.log.HoodieLogFormat import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType import org.apache.hudi.common.table.timeline.HoodieTimeline import org.apache.hudi.common.testutils.HoodieTestUtils @@ -745,23 +745,25 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase { val schema = new TableSchemaResolver(metaClient).getTableSchema for (i <- 0 until expectedNumLogFile) { - val logReader = new HoodieLogFileReader( - metaClient.getStorage, new HoodieLogFile(logFilePathList.get(i)), - schema, 1024 * 1024, false, false, - "id", null) - assertTrue(logReader.hasNext) - val logBlockHeader = logReader.next().getLogBlockHeader - assertTrue(logBlockHeader.containsKey(HeaderMetadataType.SCHEMA)) - if (isPartial) { - assertTrue(logBlockHeader.containsKey(HeaderMetadataType.IS_PARTIAL)) - assertTrue(logBlockHeader.get(HeaderMetadataType.IS_PARTIAL).toBoolean) - } else { - assertFalse(logBlockHeader.containsKey(HeaderMetadataType.IS_PARTIAL)) + val logReader = HoodieLogFormat.newReader( + metaClient, new HoodieLogFile(logFilePathList.get(i)), schema) + try { + assertTrue(logReader.hasNext) + val logBlockHeader = logReader.next().getLogBlockHeader + assertTrue(logBlockHeader.containsKey(HeaderMetadataType.SCHEMA)) + if (isPartial) { + assertTrue(logBlockHeader.containsKey(HeaderMetadataType.IS_PARTIAL)) + assertTrue(logBlockHeader.get(HeaderMetadataType.IS_PARTIAL).toBoolean) + } else { + assertFalse(logBlockHeader.containsKey(HeaderMetadataType.IS_PARTIAL)) + } + val actualSchema = HoodieSchema.parse(logBlockHeader.get(HeaderMetadataType.SCHEMA)) + val expectedSchema = HoodieSchemaUtils.addMetadataFields(HoodieSchemaUtils.generateProjectionSchema( + schema, changedFields(i).asJava), false) + assertEquals(expectedSchema, actualSchema) + } finally { + logReader.close() } - val actualSchema = HoodieSchema.parse(logBlockHeader.get(HeaderMetadataType.SCHEMA)) - val expectedSchema = HoodieSchemaUtils.addMetadataFields(HoodieSchemaUtils.generateProjectionSchema( - schema, changedFields(i).asJava), false) - assertEquals(expectedSchema, actualSchema) } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestSecondaryIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestSecondaryIndex.scala index 5d592e2137121..81c8127496af5 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestSecondaryIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestSecondaryIndex.scala @@ -265,6 +265,7 @@ class TestSecondaryIndex extends HoodieSparkSqlTestBase { val tableName = generateTableName val basePath = s"${tmp.getCanonicalPath}/$tableName" withSQLConf("hoodie.embed.timeline.server" -> "false", + "hoodie.write.table.version" -> "8", "hoodie.spark.sql.insert.into.operation" -> "upsert") { // Create table with version 8 @@ -311,6 +312,7 @@ class TestSecondaryIndex extends HoodieSparkSqlTestBase { )) // Upgrade table to version 9 and verify secondary indexes are dropped + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. withSparkSqlSessionConfig(s"hoodie.write.table.version" -> "9") { // Update a record to trigger version upgrade spark.sql(s"insert into $tableName values(1, 'a1', 11, 1001)") @@ -399,91 +401,94 @@ class TestSecondaryIndex extends HoodieSparkSqlTestBase { test("Test Secondary Index With Updates Compaction Clustering Deletes") { withTempDir { tmp => - val tableName = generateTableName - val basePath = s"${tmp.getCanonicalPath}/$tableName" - // Step 1: Initial Insertion of Records - val dataGen = new HoodieTestDataGenerator() - val hudiOpts: Map[String, String] = loadInitialBatchAndCreateSecondaryIndex(tableName, basePath, dataGen) - - // Verify initial state of secondary index - val initialKeys = spark.sql(s"select _row_key from $tableName limit 5").collect().map(_.getString(0)) - validateSecondaryIndex(basePath, tableName, initialKeys) - val initialRecordsCount = spark.sql(s"select _row_key from $tableName").count() - - // Step 3: Perform Update Operations on Subset of Records - var updateRecords = recordsToStrings(dataGen.generateUniqueUpdates(getInstantTime, 10, HoodieTestDataGenerator.TRIP_FLATTENED_SCHEMA)).asScala - var updateDf = spark.read.json(spark.sparkContext.parallelize(updateRecords.toSeq, 2)) - updateDf.write.format("hudi") - .options(hudiOpts) - .option(OPERATION.key, UPSERT_OPERATION_OPT_VAL) - .mode(SaveMode.Append) - .save(basePath) - // Verify secondary index after updates - var updateKeys = updateDf.select("_row_key").collect().map(_.getString(0)) - validateSecondaryIndex(basePath, tableName, updateKeys) - - // Step 4: Trigger Compaction with this update as the compaction frequency is set to 3 commits - updateRecords = recordsToStrings(dataGen.generateUniqueUpdates(getInstantTime, 10, HoodieTestDataGenerator.TRIP_FLATTENED_SCHEMA)).asScala - updateDf = spark.read.json(spark.sparkContext.parallelize(updateRecords.toSeq, 2)) - updateDf.write.format("hudi") - .options(hudiOpts) - .option(OPERATION.key, UPSERT_OPERATION_OPT_VAL) - .mode(SaveMode.Append) - .save(basePath) - // Verify compaction - var metaClient = HoodieTableMetaClient.builder() - .setBasePath(basePath) - .setConf(HoodieTestUtils.getDefaultStorageConf) - .build() - assertTrue(metaClient.getActiveTimeline.getCommitTimeline.filterCompletedInstants.lastInstant.isPresent) - // Verify secondary index after compaction - updateKeys = updateDf.select("_row_key").collect().map(_.getString(0)) - validateSecondaryIndex(basePath, tableName, updateKeys) - // Verify count of records - assertEquals(initialRecordsCount, spark.sql(s"select _row_key from $tableName").count()) + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. + withSparkSqlSessionConfig(s"hoodie.write.table.version" -> "9") { + val tableName = generateTableName + val basePath = s"${tmp.getCanonicalPath}/$tableName" + // Step 1: Initial Insertion of Records + val dataGen = new HoodieTestDataGenerator() + val hudiOpts: Map[String, String] = loadInitialBatchAndCreateSecondaryIndex(tableName, basePath, dataGen) - // Step 5: Trigger Clustering with this update as the clustering frequency is set to 4 commits - updateRecords = recordsToStrings(dataGen.generateUniqueUpdates(getInstantTime, 10, HoodieTestDataGenerator.TRIP_FLATTENED_SCHEMA)).asScala - updateDf = spark.read.json(spark.sparkContext.parallelize(updateRecords.toSeq, 2)) - updateDf.write.format("hudi") - .options(hudiOpts) - .option(OPERATION.key, UPSERT_OPERATION_OPT_VAL) - .mode(SaveMode.Append) - .save(basePath) - // Verify clustering - metaClient = HoodieTableMetaClient.reload(metaClient) - assertTrue(metaClient.getActiveTimeline.getCompletedReplaceTimeline.lastInstant.isPresent) - // Verify secondary index after clustering - updateKeys = updateDf.select("_row_key").collect().map(_.getString(0)) - validateSecondaryIndex(basePath, tableName, updateKeys) + // Verify initial state of secondary index + val initialKeys = spark.sql(s"select _row_key from $tableName limit 5").collect().map(_.getString(0)) + validateSecondaryIndex(basePath, tableName, initialKeys) + val initialRecordsCount = spark.sql(s"select _row_key from $tableName").count() - // Step 6: Perform Deletes on Records and Validate Secondary Index - val deleteKeys = initialKeys.take(1) // pick a subset of keys to delete - val deleteDf = spark.read.format("hudi").load(basePath).filter(s"_row_key in ('${deleteKeys.mkString("','")}')") - deleteDf.write.format("hudi") - .options(hudiOpts) - .option(OPERATION.key, DELETE_OPERATION_OPT_VAL) - .mode(SaveMode.Append) - .save(basePath) - // Verify secondary index for deletes - validateSecondaryIndex(basePath, tableName, deleteKeys, hasDeleteKeys = true) - // Verify for non deleted keys - val nonDeletedKeys = initialKeys.diff(deleteKeys) - validateSecondaryIndex(basePath, tableName, nonDeletedKeys) - - // Step 7: Final Update and Validation - val finalUpdateRecords = recordsToStrings(dataGen.generateUniqueUpdates(getInstantTime, 10, HoodieTestDataGenerator.TRIP_FLATTENED_SCHEMA)).asScala - val finalUpdateDf = spark.read.json(spark.sparkContext.parallelize(finalUpdateRecords.toSeq, 2)) - finalUpdateDf.write.format("hudi") - .options(hudiOpts) - .option(OPERATION.key, UPSERT_OPERATION_OPT_VAL) - .mode(SaveMode.Append) - .save(basePath) - // Verify secondary index after final updates - val finalUpdateKeys = finalUpdateDf.select("_row_key").collect().map(_.getString(0)) - validateSecondaryIndex(basePath, tableName, nonDeletedKeys) - validateSecondaryIndex(basePath, tableName, finalUpdateKeys) - dataGen.close() + // Step 3: Perform Update Operations on Subset of Records + var updateRecords = recordsToStrings(dataGen.generateUniqueUpdates(getInstantTime, 10, HoodieTestDataGenerator.TRIP_FLATTENED_SCHEMA)).asScala + var updateDf = spark.read.json(spark.sparkContext.parallelize(updateRecords.toSeq, 2)) + updateDf.write.format("hudi") + .options(hudiOpts) + .option(OPERATION.key, UPSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + // Verify secondary index after updates + var updateKeys = updateDf.select("_row_key").collect().map(_.getString(0)) + validateSecondaryIndex(basePath, tableName, updateKeys) + + // Step 4: Trigger Compaction with this update as the compaction frequency is set to 3 commits + updateRecords = recordsToStrings(dataGen.generateUniqueUpdates(getInstantTime, 10, HoodieTestDataGenerator.TRIP_FLATTENED_SCHEMA)).asScala + updateDf = spark.read.json(spark.sparkContext.parallelize(updateRecords.toSeq, 2)) + updateDf.write.format("hudi") + .options(hudiOpts) + .option(OPERATION.key, UPSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + // Verify compaction + var metaClient = HoodieTableMetaClient.builder() + .setBasePath(basePath) + .setConf(HoodieTestUtils.getDefaultStorageConf) + .build() + assertTrue(metaClient.getActiveTimeline.getCommitTimeline.filterCompletedInstants.lastInstant.isPresent) + // Verify secondary index after compaction + updateKeys = updateDf.select("_row_key").collect().map(_.getString(0)) + validateSecondaryIndex(basePath, tableName, updateKeys) + // Verify count of records + assertEquals(initialRecordsCount, spark.sql(s"select _row_key from $tableName").count()) + + // Step 5: Trigger Clustering with this update as the clustering frequency is set to 4 commits + updateRecords = recordsToStrings(dataGen.generateUniqueUpdates(getInstantTime, 10, HoodieTestDataGenerator.TRIP_FLATTENED_SCHEMA)).asScala + updateDf = spark.read.json(spark.sparkContext.parallelize(updateRecords.toSeq, 2)) + updateDf.write.format("hudi") + .options(hudiOpts) + .option(OPERATION.key, UPSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + // Verify clustering + metaClient = HoodieTableMetaClient.reload(metaClient) + assertTrue(metaClient.getActiveTimeline.getCompletedReplaceTimeline.lastInstant.isPresent) + // Verify secondary index after clustering + updateKeys = updateDf.select("_row_key").collect().map(_.getString(0)) + validateSecondaryIndex(basePath, tableName, updateKeys) + + // Step 6: Perform Deletes on Records and Validate Secondary Index + val deleteKeys = initialKeys.take(1) // pick a subset of keys to delete + val deleteDf = spark.read.format("hudi").load(basePath).filter(s"_row_key in ('${deleteKeys.mkString("','")}')") + deleteDf.write.format("hudi") + .options(hudiOpts) + .option(OPERATION.key, DELETE_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + // Verify secondary index for deletes + validateSecondaryIndex(basePath, tableName, deleteKeys, hasDeleteKeys = true) + // Verify for non deleted keys + val nonDeletedKeys = initialKeys.diff(deleteKeys) + validateSecondaryIndex(basePath, tableName, nonDeletedKeys) + + // Step 7: Final Update and Validation + val finalUpdateRecords = recordsToStrings(dataGen.generateUniqueUpdates(getInstantTime, 10, HoodieTestDataGenerator.TRIP_FLATTENED_SCHEMA)).asScala + val finalUpdateDf = spark.read.json(spark.sparkContext.parallelize(finalUpdateRecords.toSeq, 2)) + finalUpdateDf.write.format("hudi") + .options(hudiOpts) + .option(OPERATION.key, UPSERT_OPERATION_OPT_VAL) + .mode(SaveMode.Append) + .save(basePath) + // Verify secondary index after final updates + val finalUpdateKeys = finalUpdateDf.select("_row_key").collect().map(_.getString(0)) + validateSecondaryIndex(basePath, tableName, nonDeletedKeys) + validateSecondaryIndex(basePath, tableName, finalUpdateKeys) + dataGen.close() + } } } @@ -690,7 +695,11 @@ class TestSecondaryIndex extends HoodieSparkSqlTestBase { private def loadInitialBatchAndCreateSecondaryIndex(tableName: String, basePath: String, dataGen: HoodieTestDataGenerator, numInserts: Integer = 50) = { val initialRecords = recordsToStrings(dataGen.generateInserts(getInstantTime, numInserts, true)).asScala val initialDf = spark.read.json(spark.sparkContext.parallelize(initialRecords.toSeq, 2)) - val hudiOpts = commonOpts ++ Map(TABLE_TYPE.key -> "MERGE_ON_READ", HoodieWriteConfig.TBL_NAME.key -> tableName) + val hudiOpts = commonOpts ++ Map( + TABLE_TYPE.key -> "MERGE_ON_READ", + HoodieWriteConfig.TBL_NAME.key -> tableName, + // todo remove this option after https://github.com/apache/hudi/issues/19090 resolved. + HoodieWriteConfig.WRITE_TABLE_VERSION.key() -> "9") initialDf.write.format("hudi") .options(hudiOpts) .option(OPERATION.key, INSERT_OPERATION_OPT_VAL) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala index 78a100cf3150b..823f2cb7896d4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala @@ -157,6 +157,7 @@ class TestUpgradeOrDowngradeProcedure extends HoodieSparkProcedureTestBase { val tableName = generateTableName val tablePath = s"${tmp.getCanonicalPath}/$tableName" // create table + // todo remove option 'hoodie.write.table.version' after https://github.com/apache/hudi/issues/19090 resolved. spark.sql( s""" |create table $tableName ( @@ -169,7 +170,8 @@ class TestUpgradeOrDowngradeProcedure extends HoodieSparkProcedureTestBase { | options ( | type = 'mor', | primaryKey = 'id', - | preCombineField = 'ts' + | preCombineField = 'ts', + | hoodie.write.table.version = '9' | ) """.stripMargin) withSQLConf( diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java index fede9443ced2a..fcbe4d71c0967 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java @@ -1654,13 +1654,13 @@ Pair hasCommittedLogFiles( for (String logFilePathStr : logFilePathSet) { HoodieLogFormat.Reader reader = null; try { - HoodieSchema readerSchema = TableSchemaResolver.readSchemaFromLogFile(storage, new StoragePath(logFilePathStr)); + HoodieSchema readerSchema = TableSchemaResolver.readSchemaFromLogFile(metaClient, new StoragePath(logFilePathStr)); if (readerSchema == null) { log.warn("Cannot read schema from log file {}. Skip the check as it's likely being written by an inflight instant.", logFilePathStr); continue; } reader = - HoodieLogFormat.newReader(storage, new HoodieLogFile(logFilePathStr), readerSchema, false); + HoodieLogFormat.newReader(metaClient, new HoodieLogFile(logFilePathStr), readerSchema, false); // read the avro blocks if (reader.hasNext()) { HoodieLogBlock block = reader.next(); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java index 24ee11c8ba1ec..e5e4e1751048a 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java @@ -918,6 +918,7 @@ private void prepareTimelineAndValidate(HoodieTableMetaClient metaClient, HoodieTimeline timeline = new ActiveTimelineV2(); timeline.setInstants(instantList); when(metaClient.getCommitsTimeline()).thenReturn(timeline); + when(metaClient.getStorage()).thenReturn(storage); assertEquals(expected, validator.hasCommittedLogFiles( storage, new HashSet<>(Collections.singletonList(logFile.getPath().toString())),