Skip to content

[WIP][ISSUE-4346] Support Spark 3.5 #4354

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
QiangCai wants to merge 5 commits into
base: master
Choose a base branch
from
Open

[WIP][ISSUE-4346] Support Spark 3.5 #4354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5d62af6
[WIP][ISSUE-4346] Support Spark 3.5
QiangCai Apr 10, 2024
2179eaa
[WIP][ISSUE-4346] Support Spark 3.5
QiangCai Jun 10, 2024
5ba93de
fix sparksession example
QiangCai Jun 16, 2024
2f1bea0
fix hive test case
QiangCai Jun 16, 2024
122a544
fix sdk test case
QiangCai Jun 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions assembly/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,12 @@
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<artifactId>carbondata-spark</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-mv-plan_${spark.binary.version}</artifactId>
<artifactId>carbondata-mv-plan</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
Expand Down
3 changes: 2 additions & 1 deletion common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>

Expand Down
43 changes: 37 additions & 6 deletions core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,38 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.16.0</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.13.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0.1</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.23.0</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
Expand All @@ -67,7 +98,7 @@
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
<version>1.3.2-2</version>
<version>1.5.5-4</version>
</dependency>
<dependency>
<groupId>org.jmockit</groupId>
Expand All @@ -82,7 +113,7 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.7</version>
<version>3.6.3</version>
<exclusions>
<exclusion>
<groupId>jline</groupId>
Expand All @@ -97,7 +128,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.roaringbitmap</groupId>
Expand All @@ -112,12 +143,12 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.17.Final</version>
<version>4.1.96.Final</version>
</dependency>
<dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
<version>1.4.0</version>
<version>1.8.0</version>
</dependency>
</dependencies>

Expand Down
12 changes: 0 additions & 12 deletions core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2659,18 +2659,6 @@ private CarbonCommonConstants() {

public static final String CARBON_SDK_EMPTY_METADATA_PATH = "emptyMetadataFolder";

/**
* Property to identify if the spark version is above 3.x version
*/
public static final String CARBON_SPARK_VERSION_SPARK3 = "carbon.spark.version.spark3";

public static final String CARBON_SPARK_VERSION_SPARK3_DEFAULT = "false";

/**
* Carbon Spark 3.x supported data file written version
*/
public static final String CARBON_SPARK3_VERSION = "2.2.0";

/**
* This property is to enable the min max pruning of target carbon table based on input/source
* data
Expand Down
2 changes: 1 addition & 1 deletion ...rg/apache/carbondata/core/datastore/chunk/reader/dimension/v3/DimensionChunkReaderV3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
import org.apache.carbondata.format.DataChunk3;
import org.apache.carbondata.format.Encoding;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;

/**
* Dimension column V3 Reader class which will be used to read and uncompress
Expand Down
2 changes: 1 addition & 1 deletion ...va/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/MeasureChunkReaderV3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
import org.apache.carbondata.format.DataChunk3;
import org.apache.carbondata.format.Encoding;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;

/**
* Measure column V3 Reader class which will be used to read and uncompress
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
import org.apache.carbondata.core.util.ThreadLocalSessionInfo;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
import org.apache.carbondata.core.util.CarbonProperties;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.path.CarbonTablePath;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;

/**
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/carbondata/core/metadata/schema/table/IndexSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import static org.apache.carbondata.core.constants.CarbonCommonConstants.INDEX_COLUMNS;

import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;

/**
* It is the new schema of index and it has less fields compare to {{@link IndexSchema}}
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public void openThriftReader(String filePath) throws IOException {
*
* @param fileData
*/
public void openThriftReader(byte[] fileData) {
public void openThriftReader(byte[] fileData) throws IOException {
thriftReader = new ThriftReader(fileData);
}

Expand Down
2 changes: 1 addition & 1 deletion .../main/java/org/apache/carbondata/core/scan/expression/conditional/ImplicitExpression.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.util.path.CarbonTablePath;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;

/**
* Custom class to handle filter values for Implicit filter
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,9 @@
import com.google.gson.GsonBuilder;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.input.ClassLoaderObjectInputStream;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
Expand Down
46 changes: 3 additions & 43 deletions core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,50 +544,10 @@ private static long createTimeInstant(String dimensionValue, String dateFormat)
}

private static Object parseTimestamp(String dimensionValue, String dateFormat) {
Date dateToStr;
DateFormat dateFormatter = null;
long timeValue;
try {
if (Boolean.parseBoolean(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_SPARK_VERSION_SPARK3,
CarbonCommonConstants.CARBON_SPARK_VERSION_SPARK3_DEFAULT))) {
try {
return createTimeInstant(dimensionValue, dateFormat.trim());
} catch (DateTimeParseException e) {
throw new NumberFormatException(e.getMessage());
}
}
if (null != dateFormat && !dateFormat.trim().isEmpty()) {
dateFormatter = new SimpleDateFormat(dateFormat);
dateFormatter.setLenient(false);
} else {
dateFormatter = timestampFormatter.get();
}
dateToStr = dateFormatter.parse(dimensionValue);
timeValue = dateToStr.getTime();
validateTimeStampRange(timeValue);
return timeValue;
} catch (ParseException e) {
// If the parsing fails, try to parse again with setLenient to true if the property is set
// (example: 1941-03-15 00:00:00 is invalid data and will fail to parse in Asia/Shanghai zone
// as DST is observed and clocks were turned forward 1 hour to 1941-03-15 01:00:00)
if (CarbonProperties.getInstance().isSetLenientEnabled()) {
try {
dateFormatter.setLenient(true);
dateToStr = dateFormatter.parse(dimensionValue);
timeValue = dateToStr.getTime();
validateTimeStampRange(timeValue);
LOGGER.info("Parsed data with lenience as true, setting back to default mode");
return timeValue;
} catch (ParseException ex) {
LOGGER.info("Failed to parse data with lenience as true, setting back to default mode");
throw new NumberFormatException(ex.getMessage());
} finally {
dateFormatter.setLenient(false);
}
} else {
throw new NumberFormatException(e.getMessage());
}
return createTimeInstant(dimensionValue, dateFormat.trim());
} catch (DateTimeParseException e) {
throw new NumberFormatException(e.getMessage());
}
}

Expand Down
2 changes: 1 addition & 1 deletion docs/sdk-guide.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ import org.apache.carbondata.sdk.file.CarbonWriter;
import org.apache.carbondata.core.metadata.datatype.Field;

import org.apache.avro.generic.GenericData;
import org.apache.commons.lang.CharEncoding;
import org.apache.commons.lang3.CharEncoding;

import tech.allegro.schema.json2avro.converter.JsonAvroConverter;

Expand Down
2 changes: 1 addition & 1 deletion examples/flink/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<artifactId>carbondata-spark</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
Expand Down
25 changes: 2 additions & 23 deletions examples/spark/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<artifactId>carbondata-spark</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
Expand Down Expand Up @@ -157,6 +157,7 @@
<plugin>
<groupId>com.ning.maven.plugins</groupId>
<artifactId>maven-duplicate-finder-plugin</artifactId>
<version>1.0.9</version>
<configuration>
<skip>true</skip>
</configuration>
Expand Down Expand Up @@ -198,27 +199,5 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-2.3</id>
<properties>
<spark.binary.version>2.3</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-2.4</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.binary.version>2.4</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-3.1</id>
<properties>
<spark.binary.version>3.1</spark.binary.version>
<dep.jackson.version>2.10.0</dep.jackson.version>
</properties>
</profile>
</profiles>
</project>
3 changes: 2 additions & 1 deletion geo/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
Expand Down Expand Up @@ -126,6 +126,7 @@
<plugin>
<groupId>com.ning.maven.plugins</groupId>
<artifactId>maven-duplicate-finder-plugin</artifactId>
<version>1.0.9</version>
<configuration>
<skip>true</skip>
</configuration>
Expand Down
2 changes: 1 addition & 1 deletion geo/src/main/java/org/apache/carbondata/geo/scan/expression/PolygonListExpression.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import org.apache.carbondata.geo.GeoHashUtils;
import org.apache.carbondata.geo.GeoOperationType;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;

/**
* InPolygonList expression processor. It inputs the InPolygonList string to the Geo
Expand Down
2 changes: 1 addition & 1 deletion hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
import org.apache.carbondata.hadoop.CarbonInputSplit;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
Expand Down
2 changes: 1 addition & 1 deletion hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
import org.apache.carbondata.hadoop.CarbonInputSplit;

import com.google.common.collect.Sets;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
Expand Down
2 changes: 1 addition & 1 deletion index/bloom/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0.1</version>
<version>20.0</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
Expand Down
Loading