Skip to content

Commit 5602fbf

Browse files
senthhMaxGekk
authored andcommitted
[SPARK-51421][SQL] Get seconds of TIME datatype
### What changes were proposed in this pull request? This PR adds support for extracting the second component from TIME (TimeType) values in Spark SQL. For example: ``` scala> spark.sql("SELECT SECOND(TIME'13:59:45.99')").show() +--------------------------+ |second(TIME '13:59:45.99')| +--------------------------+ | 45| +--------------------------+ scala> spark.sql("select second(cast('12:00:01.123' as time(4)))").show(false) +-------------------------------------+ |second(CAST(12:00:01.123 AS TIME(4)))| +-------------------------------------+ |1 | +-------------------------------------+ ``` ### Why are the changes needed? Spark previously supported second() for only TIMESTAMP type values. TIME support was missing, leading to implicit casting attempt to TIMESTAMP, which was incorrect. This PR ensures that second(TIME'HH:MM:SS.######') behaves correctly without unnecessary type coercion. ### Does this PR introduce _any_ user-facing change? Yes - Before this PR, calling second(TIME'HH:MM:SS.######') resulted in a type mismatch error or an implicit cast attempt to TIMESTAMP, which was incorrect. - With this PR, second(TIME'HH:MM:SS.######') now works correctly for TIME values without implicit casting. - Users can now extract the second component from TIME values natively. ### How was this patch tested? By running new tests: ```$ build/sbt "test:testOnly *TimeExpressionsSuite"``` ### Was this patch authored or co-authored using generative AI tooling? No Closes #50525 from senthh/getSeconds. Authored-by: senthh <[email protected]> Signed-off-by: Max Gekk <[email protected]>
1 parent f4ce0ab commit 5602fbf

File tree

5 files changed

+126
-3
lines changed

5 files changed

+126
-3
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -648,7 +648,7 @@ object FunctionRegistry {
648648
expression[NextDay]("next_day"),
649649
expression[Now]("now"),
650650
expression[Quarter]("quarter"),
651-
expression[Second]("second"),
651+
expressionBuilder("second", SecondExpressionBuilder),
652652
expression[ParseToTimestamp]("to_timestamp"),
653653
expression[ParseToDate]("to_date"),
654654
expression[ToTime]("to_time"),

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
2525
import org.apache.spark.sql.catalyst.util.TimeFormatter
2626
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
2727
import org.apache.spark.sql.internal.types.StringTypeWithCollation
28-
import org.apache.spark.sql.types.{AbstractDataType, IntegerType, ObjectType, TimeType}
28+
import org.apache.spark.sql.types.{AbstractDataType, IntegerType, ObjectType, TimeType, TypeCollection}
2929
import org.apache.spark.unsafe.types.UTF8String
3030

3131
/**
@@ -290,3 +290,62 @@ object HourExpressionBuilder extends ExpressionBuilder {
290290
}
291291
}
292292
}
293+
294+
case class SecondsOfTime(child: Expression)
295+
extends RuntimeReplaceable
296+
with ExpectsInputTypes {
297+
298+
override def replacement: Expression = StaticInvoke(
299+
classOf[DateTimeUtils.type],
300+
IntegerType,
301+
"getSecondsOfTime",
302+
Seq(child),
303+
Seq(child.dataType)
304+
)
305+
306+
override def inputTypes: Seq[AbstractDataType] =
307+
Seq(TypeCollection(TimeType.MIN_PRECISION to TimeType.MAX_PRECISION map TimeType: _*))
308+
309+
override def children: Seq[Expression] = Seq(child)
310+
311+
override def prettyName: String = "second"
312+
313+
override protected def withNewChildrenInternal(
314+
newChildren: IndexedSeq[Expression]): Expression = {
315+
copy(child = newChildren.head)
316+
}
317+
}
318+
319+
@ExpressionDescription(
320+
usage = """
321+
_FUNC_(expr) - Returns the second component of the given expression.
322+
323+
If `expr` is a TIMESTAMP or a string that can be cast to timestamp,
324+
it returns the second of that timestamp.
325+
If `expr` is a TIME type (since 4.1.0), it returns the second of the time-of-day.
326+
""",
327+
examples = """
328+
Examples:
329+
> SELECT _FUNC_('2018-02-14 12:58:59');
330+
59
331+
> SELECT _FUNC_(TIME'13:25:59.999999');
332+
59
333+
""",
334+
since = "1.5.0",
335+
group = "datetime_funcs")
336+
object SecondExpressionBuilder extends ExpressionBuilder {
337+
override def build(name: String, expressions: Seq[Expression]): Expression = {
338+
if (expressions.isEmpty) {
339+
throw QueryCompilationErrors.wrongNumArgsError(name, Seq("> 0"), expressions.length)
340+
} else {
341+
val child = expressions.head
342+
child.dataType match {
343+
case _: TimeType =>
344+
SecondsOfTime(child)
345+
case _ =>
346+
Second(child)
347+
}
348+
}
349+
}
350+
}
351+

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,12 @@ object DateTimeUtils extends SparkDateTimeUtils {
135135
getLocalDateTime(micros, zoneId).getSecond
136136
}
137137

138+
/**
139+
* Returns the second value of a given TIME (TimeType) value.
140+
*/
141+
def getSecondsOfTime(micros: Long): Int = {
142+
microsToLocalTime(micros).getSecond
143+
}
138144
/**
139145
* Returns the seconds part and its fractional part with microseconds.
140146
*/

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,4 +168,62 @@ class TimeExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
168168
checkConsistencyBetweenInterpretedAndCodegen(
169169
(child: Expression) => MinutesOfTime(child).replacement, TimeType())
170170
}
171+
172+
test("SecondExpressionBuilder") {
173+
// Empty expressions list
174+
checkError(
175+
exception = intercept[AnalysisException] {
176+
SecondExpressionBuilder.build("second", Seq.empty)
177+
},
178+
condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
179+
parameters = Map(
180+
"functionName" -> "`second`",
181+
"expectedNum" -> "> 0",
182+
"actualNum" -> "0",
183+
"docroot" -> SPARK_DOC_ROOT)
184+
)
185+
186+
// test TIME-typed child should build SecondsOfTime
187+
val timeExpr = Literal(localTime(12, 58, 59), TimeType())
188+
val builtExprForTime = SecondExpressionBuilder.build("second", Seq(timeExpr))
189+
assert(builtExprForTime.isInstanceOf[SecondsOfTime])
190+
assert(builtExprForTime.asInstanceOf[SecondsOfTime].child eq timeExpr)
191+
192+
// test non TIME-typed child should build second
193+
val tsExpr = Literal("2007-09-03 10:45:23")
194+
val builtExprForTs = SecondExpressionBuilder.build("second", Seq(tsExpr))
195+
assert(builtExprForTs.isInstanceOf[Second])
196+
assert(builtExprForTs.asInstanceOf[Second].child eq tsExpr)
197+
}
198+
199+
test("Second with TIME type") {
200+
// A few test times in microseconds since midnight:
201+
// time in microseconds -> expected second
202+
val testTimes = Seq(
203+
localTime() -> 0,
204+
localTime(1) -> 0,
205+
localTime(0, 59) -> 0,
206+
localTime(14, 30) -> 0,
207+
localTime(12, 58, 59) -> 59,
208+
localTime(23, 0, 1) -> 1,
209+
localTime(23, 59, 59, 999999) -> 59
210+
)
211+
212+
// Create a literal with TimeType() for each test microsecond value
213+
// evaluate SecondsOfTime(...), and check that the result matches the expected second.
214+
testTimes.foreach { case (micros, expectedSecond) =>
215+
checkEvaluation(
216+
SecondsOfTime(Literal(micros, TimeType())),
217+
expectedSecond)
218+
}
219+
220+
// Verify NULL handling
221+
checkEvaluation(
222+
SecondsOfTime(Literal.create(null, TimeType(TimeType.MICROS_PRECISION))),
223+
null
224+
)
225+
226+
checkConsistencyBetweenInterpretedAndCodegen(
227+
(child: Expression) => SecondsOfTime(child).replacement, TimeType())
228+
}
171229
}

sql/core/src/test/resources/sql-functions/sql-expression-schema.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@
286286
| org.apache.spark.sql.catalyst.expressions.SchemaOfJson | schema_of_json | SELECT schema_of_json('[{"col":0}]') | struct<schema_of_json([{"col":0}]):string> |
287287
| org.apache.spark.sql.catalyst.expressions.SchemaOfXml | schema_of_xml | SELECT schema_of_xml('<p><a>1</a></p>') | struct<schema_of_xml(<p><a>1</a></p>):string> |
288288
| org.apache.spark.sql.catalyst.expressions.Sec | sec | SELECT sec(0) | struct<SEC(0):double> |
289-
| org.apache.spark.sql.catalyst.expressions.Second | second | SELECT second('2009-07-30 12:58:59') | struct<second(2009-07-30 12:58:59):int> |
289+
| org.apache.spark.sql.catalyst.expressions.SecondExpressionBuilder | second | SELECT second('2018-02-14 12:58:59') | struct<second(2018-02-14 12:58:59):int> |
290290
| org.apache.spark.sql.catalyst.expressions.SecondsToTimestamp | timestamp_seconds | SELECT timestamp_seconds(1230219000) | struct<timestamp_seconds(1230219000):timestamp> |
291291
| org.apache.spark.sql.catalyst.expressions.Sentences | sentences | SELECT sentences('Hi there! Good morning.') | struct<sentences(Hi there! Good morning., , ):array<array<string>>> |
292292
| org.apache.spark.sql.catalyst.expressions.Sequence | sequence | SELECT sequence(1, 5) | struct<sequence(1, 5):array<int>> |

0 commit comments

Comments
 (0)