[SPARK-52567][SQL][HIVE][TEST] Refactor Hive_2_1_DDLSuite

### What changes were proposed in this pull request?

Hive_2_1_DDLSuite was added in SPARK-21617 (2.3.0), in those days, the built-in Hive version was 1.2.1.

> A separate set of DDL tests that uses Hive 2.1 libraries, which behave a little differently from the built-in ones.

After a closer look, the real difference is, Hive enables `hive.metastore.disallow.incompatible.col.type.changes` since HIVE-12320 (2.0.0)

Now we upgraded the built-in Hive to 2.3.10, the comments become misleading and we can simply use the global `TestHiveSingleton` with overriding `hive.metastore.disallow.incompatible.col.type.changes` to achieve the same test purpose.

### Why are the changes needed?

Simplify and speed up the test, and update the outdated comments for clarity.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

```
sbt:spark-hive> testOnly *HiveIncompatibleColTypeChangeSuite
...
[info] Run completed in 11 seconds, 429 milliseconds.
[info] Total number of tests run: 4
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 4, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
[success] Total time: 18 s, completed Jun 24, 2025, 11:26:07 PM
```

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #51271 from pan3793/SPARK-52567.

Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: pan3793

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala renamed to sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveIncompatibleColTypeChangeSuite.scala

@@ -17,49 +17,23 @@
 
 package org.apache.spark.sql.hive.execution
 
-import org.apache.hadoop.conf.Configuration
-
-import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.launcher.SparkLauncher
+import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
 import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.sql.types._
-import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
-import org.apache.spark.util.Utils
 
 /**
- * A separate set of DDL tests that uses Hive 2.1 libraries, which behave a little differently
- * from the built-in ones.
+ * A separate set of Hive DDL tests when setting
+ * `hive.metastore.disallow.incompatible.col.type.changes=true`
  */
-@SlowHiveTest
-@ExtendedHiveTest
-class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton {
-
-  // Create a custom HiveExternalCatalog instance with the desired configuration. We cannot
-  // use SparkSession here since there's already an active on managed by the TestHive object.
-  private var catalog = {
-    val warehouse = Utils.createTempDir()
-    val metastore = Utils.createTempDir()
-    metastore.delete()
-    val sparkConf = new SparkConf()
-      .set(SparkLauncher.SPARK_MASTER, "local")
-      .set(WAREHOUSE_PATH.key, warehouse.toURI().toString())
-      .set(CATALOG_IMPLEMENTATION.key, "hive")
-      .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.1")
-      .set(HiveUtils.HIVE_METASTORE_JARS.key, "maven")
+class HiveIncompatibleColTypeChangeSuite extends SparkFunSuite with TestHiveSingleton {
 
-    val hadoopConf = new Configuration()
-    hadoopConf.set("hive.metastore.warehouse.dir", warehouse.toURI().toString())
-    hadoopConf.set("javax.jdo.option.ConnectionURL",
-      s"jdbc:derby:;databaseName=${metastore.getAbsolutePath()};create=true")
-    // These options are needed since the defaults in Hive 2.1 cause exceptions with an
-    // empty metastore db.
-    hadoopConf.set("datanucleus.schema.autoCreateAll", "true")
-    hadoopConf.set("hive.metastore.schema.verification", "false")
+  private val catalog = spark.sessionState.catalog.externalCatalog
 
-    new HiveExternalCatalog(sparkConf, hadoopConf)
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    hiveClient.runSqlHive(
+      "SET hive.metastore.disallow.incompatible.col.type.changes=true")
   }
 
   override def afterEach(): Unit = {
@@ -71,7 +45,8 @@ class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton {
 
   override def afterAll(): Unit = {
     try {
-      catalog = null
+      hiveClient.runSqlHive(
+        "SET hive.metastore.disallow.incompatible.col.type.changes=false")
     } finally {
       super.afterAll()
     }
@@ -122,7 +97,7 @@ class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton {
       updatedSchema: StructType,
       hiveCompatible: Boolean = true): Unit = {
     spark.sql(createTableStmt)
-    val oldTable = spark.sessionState.catalog.externalCatalog.getTable("default", tableName)
+    val oldTable = catalog.getTable("default", tableName)
     catalog.createTable(oldTable, true)
     catalog.alterTableDataSchema("default", tableName, updatedSchema)