fix incorrect date type value when timezone is set (#1704)

Co-authored-by: birdstorm <[email protected]>
pingcap · Oct 30, 2020 · ada4c01 · ada4c01
1 parent dfbc6dd
commit ada4c01
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 5 deletions.
diff --git a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala
@@ -21,6 +21,20 @@ import org.apache.spark.sql.functions.{col, sum}
 
 class IssueTestSuite extends BaseTiSparkTest {
 
+  test("Date type deals with timezone incorrectly") {
+    tidbStmt.execute("drop table if exists t")
+    tidbStmt.execute("""
+                       |CREATE TABLE `t` (
+                       |  `id` int(11) DEFAULT NULL,
+                       |  `d` date DEFAULT NULL
+                       |) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
+                       |""".stripMargin)
+
+    tidbStmt.execute("insert into t values(1, '2020-10-25'), (2, '2020-11-20')")
+
+    explainAndRunTest(s"SELECT * FROM t")
+  }
+
   test("Scala match error when predicate includes boolean type conversion") {
     tidbStmt.execute("drop table if exists t")
     tidbStmt.execute("""

diff --git a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java
@@ -197,7 +197,7 @@ private long getTime(int rowId) {
     int month = (int) (ym % 13);
     int day = (int) (ymd & ((1 << 5) - 1));
     LocalDate date = new LocalDate(year, month, day);
-    return Math.floorDiv(date.toDate().getTime(), AbstractDateTimeType.MILLS_PER_DAY);
+    return ((DateType) type).getDays(date);
   }
   /**
    * Returns the long type value for rowId. The return value is undefined and can be anything, if

diff --git a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiChunkColumnVector.java b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiChunkColumnVector.java
@@ -144,7 +144,7 @@ private long getTime(int rowId) {
     }
     if (this.type instanceof DateType) {
       LocalDate date = new LocalDate(year, month, day);
-      return Math.floorDiv(date.toDate().getTime(), AbstractDateTimeType.MILLS_PER_DAY);
+      return ((DateType) this.type).getDays(date);
     } else if (type instanceof DateTimeType || type instanceof TimestampType) {
       // only return microsecond from epoch.
       Timestamp ts =

diff --git a/tikv-client/src/main/java/com/pingcap/tikv/types/AbstractDateTimeType.java b/tikv-client/src/main/java/com/pingcap/tikv/types/AbstractDateTimeType.java
@@ -30,7 +30,6 @@
 import org.joda.time.LocalDate;
 
 public abstract class AbstractDateTimeType extends DataType {
-  public static long MILLS_PER_DAY = 3600L * 24 * 1000;
 
   AbstractDateTimeType(InternalTypeHolder holder) {
     super(holder);

diff --git a/tikv-client/src/main/java/com/pingcap/tikv/types/DateType.java b/tikv-client/src/main/java/com/pingcap/tikv/types/DateType.java
@@ -25,9 +25,11 @@
 import com.pingcap.tikv.meta.TiColumnInfo;
 import java.sql.Date;
 import org.joda.time.DateTimeZone;
+import org.joda.time.Days;
 import org.joda.time.LocalDate;
 
 public class DateType extends AbstractDateTimeType {
+  private static final LocalDate EPOCH = new LocalDate(0);
   public static final DateType DATE = new DateType(MySQLType.TypeDate);
   public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeDate};
 
@@ -90,6 +92,16 @@ public String getName() {
     return "DATE";
   }
 
+  public int getDays(LocalDate d) {
+    // count how many days from EPOCH
+    int days = Days.daysBetween(EPOCH, d).getDays();
+    // if the timezone has negative offset, minus one day.
+    if (getTimezone().getOffset(0) < 0) {
+      days -= 1;
+    }
+    return days;
+  }
+
   /** {@inheritDoc} */
   @Override
   protected Long decodeNotNull(int flag, CodecDataInput cdi) {
@@ -98,8 +110,8 @@ protected Long decodeNotNull(int flag, CodecDataInput cdi) {
     if (date == null) {
       return null;
     }
-    // return how many days from EPOCH
-    return Math.floorDiv(date.toDate().getTime(), AbstractDateTimeType.MILLS_PER_DAY);
+
+    return (long) getDays(date);
   }
 
   @Override