Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.List;
import java.util.Set;

Expand Down Expand Up @@ -412,8 +415,27 @@ public boolean isMinorNecessary() {
}

protected boolean reachMinorInterval() {
return config.getMinorLeastInterval() >= 0
&& planTime - lastMinorOptimizingTime > config.getMinorLeastInterval();
if (config.getMinorLeastInterval() < 0) {
return false;
}
if (planTime - lastMinorOptimizingTime > config.getMinorLeastInterval()) {
return true;
}
// When minorLeastInterval is less than one day, use a cross-day fallback to ensure
// partitions with few small files still get optimized at least once per day, avoiding
// starvation caused by table-level lastMinorOptimizingTime being frequently reset by
// high-traffic partitions. See https://github.com/apache/amoro/issues/4055
long oneDayMillis = 24 * 60 * 60 * 1000L;
if (config.getMinorLeastInterval() < oneDayMillis) {
return isDifferentDay(lastMinorOptimizingTime, planTime);
}
return false;
}

private boolean isDifferentDay(long time1, long time2) {
LocalDate day1 = Instant.ofEpochMilli(time1).atZone(ZoneId.systemDefault()).toLocalDate();
LocalDate day2 = Instant.ofEpochMilli(time2).atZone(ZoneId.systemDefault()).toLocalDate();
return !day1.equals(day2);
}

protected boolean reachFullInterval() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,14 @@ private TableProperties() {}
"self-optimizing.minor.trigger.file-count";
public static final int SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT_DEFAULT = 12;

/**
* The minimum interval for triggering minor optimizing, in milliseconds. Should be less than one
* day (86400000 ms). When the interval is less than one day, a cross-day fallback mechanism
* ensures that partitions with few small files still get optimized at least once per day.
*/
public static final String SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL =
"self-optimizing.minor.trigger.interval";

public static final int SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT = 3600000; // 1 h

public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO =
Expand Down
Loading