HIVE-2942. substr on string containing UTF-8 characters produces StringIndexOutOfBoundsException (Kevin Wilfong via pauly)

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1326444 13f79535-47bb-0310-9956-ffa450edef68

Author: Paul Yang

ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java

@@ -18,14 +18,14 @@
 
 package org.apache.hadoop.hive.ql.udf;
 
+import java.util.Arrays;
+
 import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 
-import java.util.Arrays;
-
 /**
  * UDFSubstr.
  *
@@ -65,12 +65,12 @@ public Text evaluate(Text t, IntWritable pos, IntWritable len) {
       return r;
     }
 
-    int[] index = makeIndex(pos.get(), len.get(), t.getLength());
+    String s = t.toString();
+    int[] index = makeIndex(pos.get(), len.get(), s.length());
     if (index == null) {
       return r;
     }
 
-    String s = t.toString();
     r.set(s.substring(index[0], index[1]));
     return r;
   }

ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java

@@ -23,13 +23,13 @@
 import java.io.BufferedInputStream;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
-import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.FileWriter;
+import java.io.InputStreamReader;
 import java.io.PrintStream;
 import java.io.Serializable;
 import java.io.UnsupportedEncodingException;
@@ -279,7 +279,7 @@ public void addFile(File qf) throws Exception {
 
     FileInputStream fis = new FileInputStream(qf);
     BufferedInputStream bis = new BufferedInputStream(fis);
-    DataInputStream dis = new DataInputStream(bis);
+    BufferedReader br = new BufferedReader(new InputStreamReader(bis, "UTF8"));
     StringBuilder qsb = new StringBuilder();
 
     // Look for a hint to not run a test on some Hadoop versions
@@ -289,8 +289,8 @@ public void addFile(File qf) throws Exception {
     // Read the entire query
     boolean excludeQuery = false;
     String hadoopVer = ShimLoader.getMajorVersion();
-    while (dis.available() != 0) {
-      String line = dis.readLine();
+    String line;
+    while ((line = br.readLine()) != null) {
 
       // While we are reading the lines, detect whether this query wants to be
       // excluded from running because the Hadoop version is incorrect
@@ -320,7 +320,7 @@ public void addFile(File qf) throws Exception {
           "adding query " + qf.getName() + " to the set of tests to skip");
       qSkipSet.add(qf.getName());
      }
-    dis.close();
+    br.close();
   }
 
   /**

ql/src/test/queries/clientpositive/udf_substr.q

@@ -65,3 +65,11 @@ SELECT
 FROM (
    select CAST(concat(substr(value, 1, 0), 'ABC') as BINARY) as ABC from src LIMIT 1
 ) X;
+
+-- test UTF-8 substr
+SELECT
+  substr("玩", 1),
+  substr("abc 玩", 5),
+  substr("abc 玩玩玩 abc", 5),
+  substr("abc 玩玩玩 abc", 5, 3)
+FROM src LIMIT 1;

ql/src/test/results/clientpositive/udf_substr.q.out

@@ -182,3 +182,24 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 #### A masked pattern was here ####
 NULL	NULL	null	null	null	A	AB	ABC	ABC	A	AB	ABC	ABC	B	BC	BC	BC	C	C	C	C		C	C	C	C	B	BC	BC	BC	A	AB	ABC	ABC	
+PREHOOK: query: -- test UTF-8 substr
+SELECT
+  substr("玩", 1),
+  substr("abc 玩", 5),
+  substr("abc 玩玩玩 abc", 5),
+  substr("abc 玩玩玩 abc", 5, 3)
+FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- test UTF-8 substr
+SELECT
+  substr("玩", 1),
+  substr("abc 玩", 5),
+  substr("abc 玩玩玩 abc", 5),
+  substr("abc 玩玩玩 abc", 5, 3)
+FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+玩	玩	玩玩玩 abc	玩玩玩