Upgrade to hive 4.0.1

.github/trigger_files/IO_Iceberg_Integration_Tests.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run.",
-    "modification": 3
+    "modification": 4
 }

.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 3
+  "modification": 4
 }

.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml

@@ -87,28 +87,13 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
-        with:
-          java-version: |
-            8
-            11
       - name: run HCatalog IO build script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:java:io:hcatalog:build
           arguments: |
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
-      # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11
-      - name: Test HCatalog IO on Java8
-        uses: ./.github/actions/gradle-command-self-hosted-action
-        with:
-          gradle-command: :sdks:java:io:hcatalog:test
-          arguments: |
-            -PdisableSpotlessCheck=true \
-            -PdisableCheckStyle=true \
-            -Dfile.encoding=UTF-8 \
-            -PtestJavaVersion=8 \
-            -Pjava8Home=$JAVA_HOME_8_X64 \
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

.github/workflows/beam_PreCommit_Java_IOs_Direct.yml

@@ -86,10 +86,6 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
-        with:
-          java-version: |
-            8
-            11
       - name: run Java IOs PreCommit script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
@@ -98,17 +94,6 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
             -Dfile.encoding=UTF-8 \
-      # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11
-      - name: run Java8 IOs PreCommit script
-        uses: ./.github/actions/gradle-command-self-hosted-action
-        with:
-          gradle-command: :sdks:java:io:hcatalog:build
-          arguments: |
-            -PdisableSpotlessCheck=true \
-            -PdisableCheckStyle=true \
-            -Dfile.encoding=UTF-8 \
-            -PtestJavaVersion=8 \
-            -Pjava8Home=$JAVA_HOME_8_X64 \
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

sdks/java/extensions/sql/hcatalog/build.gradle

@@ -25,8 +25,8 @@ applyJavaNature(
   ],
 )
 
-def hive_version = "3.1.3"
-def netty_version = "4.1.51.Final"
+def hive_version = "4.0.1"
+def netty_version = "4.1.82.Final"  // in sync with hive-exec's dependency
 
 /*
  * We need to rely on manually specifying these evaluationDependsOn to ensure that
@@ -43,6 +43,7 @@ dependencies {
   implementation project(":sdks:java:core")
   implementation library.java.vendored_guava_32_1_2_jre
 
+  testImplementation library.java.junit
   testImplementation project(":sdks:java:io:hcatalog").sourceSets.test.output
   // Needed for HCatalogTableProvider tests,
   // they use HCat* types

sdks/java/io/hcatalog/build.gradle

@@ -38,7 +38,7 @@ def hadoopVersions = [
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
 
-def hive_version = "3.1.3"
+def hive_version = "4.0.1"
 
 dependencies {
   implementation library.java.vendored_guava_32_1_2_jre
@@ -64,6 +64,10 @@ dependencies {
   testImplementation library.java.hamcrest
   testImplementation "org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version:tests"
   testImplementation "org.apache.hive:hive-exec:$hive_version"
+  // datanucleus dependency version should be in alignment with managed dependencies of hive-standalone-metastore
+  testRuntimeOnly 'org.datanucleus:datanucleus-api-jdo:5.2.8'
+  testRuntimeOnly 'org.datanucleus:datanucleus-rdbms:5.2.10'
+  testRuntimeOnly 'org.datanucleus:javax.jdo:3.2.0-release'
   testImplementation "org.apache.hive:hive-common:$hive_version"
   testImplementation "org.apache.hive:hive-cli:$hive_version"
   testImplementation "org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version"
@@ -105,14 +109,3 @@ hadoopVersions.each { kv ->
     include '**/*Test.class'
   }
 }
-
-project.tasks.withType(Test).configureEach {
-  if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 && project.findProperty('testJavaVersion') != '8') {
-    useJUnit {
-      filter {
-        excludeTestsMatching "org.apache.beam.sdk.io.hcatalog.HCatalogIOTest"
-        excludeTestsMatching "org.apache.beam.sdk.io.hcatalog.HCatalogBeamSchemaTest"
-      }
-    }
-  }
-}

sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/test/EmbeddedMetastoreService.java

@@ -28,7 +28,7 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.DriverFactory;
 import org.apache.hadoop.hive.ql.IDriver;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
 import org.apache.hadoop.hive.ql.session.SessionState;
 
 /**
@@ -58,11 +58,11 @@ public EmbeddedMetastoreService(String baseDirPath) throws IOException {
     String testWarehouseDirPath = makePathASafeFileName(testDataDirPath + "/warehouse");
 
     hiveConf = new HiveConf(getClass());
-    hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
-    hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, "");
+    hiveConf.setVar(HiveConf.ConfVars.PRE_EXEC_HOOKS, "");
+    hiveConf.setVar(HiveConf.ConfVars.POST_EXEC_HOOKS, "");
     hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
-    hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, testWarehouseDirPath);
-    hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, true);
+    hiveConf.setVar(HiveConf.ConfVars.METASTORE_WAREHOUSE, testWarehouseDirPath);
+    hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_METADATA_QUERIES, true);
     hiveConf.setVar(
         HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
         "org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider");
@@ -75,9 +75,10 @@ public EmbeddedMetastoreService(String baseDirPath) throws IOException {
 
   /** Executes the passed query on the embedded metastore service. */
   public void executeQuery(String query) {
-    CommandProcessorResponse response = driver.run(query);
-    if (response.failed()) {
-      throw new RuntimeException(response.getException());
+    try {
+      driver.run(query);
+    } catch (CommandProcessorException e) {
+      throw new RuntimeException(e);
     }
   }
 

sdks/java/io/iceberg/build.gradle

@@ -40,7 +40,7 @@ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
 def iceberg_version = "1.6.1"
 def parquet_version = "1.12.0"
 def orc_version = "1.9.2"
-def hive_version = "3.1.3"
+def hive_version = "4.0.1"
 
 dependencies {
     implementation library.java.vendored_guava_32_1_2_jre
@@ -76,6 +76,7 @@ dependencies {
     testImplementation "org.apache.iceberg:iceberg-common:$iceberg_version"
     testImplementation ("org.apache.iceberg:iceberg-hive-metastore:$iceberg_version")
     testImplementation ("org.apache.hive:hive-metastore:$hive_version")
+    testImplementation ("org.apache.hive:hive-standalone-metastore-server:$hive_version")
     testImplementation "org.assertj:assertj-core:3.11.1"
     testRuntimeOnly ("org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version") {
         exclude group: "org.apache.hive", module: "hive-exec"

sdks/java/io/iceberg/hive/build.gradle

@@ -1,5 +1,3 @@
-import groovy.json.JsonOutput
-
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -27,7 +25,7 @@ applyJavaNature(
 description = "Apache Beam :: SDKs :: Java :: IO :: Iceberg :: Hive"
 ext.summary = "Runtime dependencies needed for Hive catalog integration."
 
-def hive_version = "3.1.3"
+def hive_version = "4.0.1"
 def hbase_version = "2.6.1-hadoop3"
 def hadoop_version = "3.4.1"
 def iceberg_version = "1.6.1"

sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/catalog/HiveCatalogIT.java

@@ -95,11 +95,12 @@ public void catalogCleanup() throws Exception {
 
   @Override
   public Map<String, Object> managedIcebergConfig(String tableId) {
-    String metastoreUri = hiveMetastoreExtension.hiveConf().getVar(HiveConf.ConfVars.METASTOREURIS);
+    String metastoreUri =
+        hiveMetastoreExtension.hiveConf().getVar(HiveConf.ConfVars.METASTORE_URIS);
 
     Map<String, String> confProperties =
         ImmutableMap.<String, String>builder()
-            .put(HiveConf.ConfVars.METASTOREURIS.varname, metastoreUri)
+            .put(HiveConf.ConfVars.METASTORE_URIS.varname, metastoreUri)
             .build();
 
     return ImmutableMap.<String, Object>builder()

sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/catalog/hiveutils/TestHiveMetastore.java

@@ -39,7 +39,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.HMSHandler;
 import org.apache.hadoop.hive.metastore.IHMSHandler;
 import org.apache.hadoop.hive.metastore.RetryingHMSHandler;
 import org.apache.hadoop.hive.metastore.TSetIpAddressProcessor;
@@ -68,10 +68,10 @@ public class TestHiveMetastore {
 
   // create the metastore handlers based on whether we're working with Hive2 or Hive3 dependencies
   // we need to do this because there is a breaking API change between Hive2 and Hive3
-  private static final DynConstructors.Ctor<HiveMetaStore.HMSHandler> HMS_HANDLER_CTOR =
+  private static final DynConstructors.Ctor<HMSHandler> HMS_HANDLER_CTOR =
       DynConstructors.builder()
-          .impl(HiveMetaStore.HMSHandler.class, String.class, Configuration.class)
-          .impl(HiveMetaStore.HMSHandler.class, String.class, HiveConf.class)
+          .impl(HMSHandler.class, String.class, Configuration.class)
+          .impl(HMSHandler.class, String.class, HiveConf.class)
           .build();
 
   private static final DynMethods.StaticMethod GET_BASE_HMS_HANDLER =
@@ -120,7 +120,7 @@ public class TestHiveMetastore {
   private HiveConf hiveConf;
   private ExecutorService executorService;
   private TServer server;
-  private HiveMetaStore.HMSHandler baseHandler;
+  private HMSHandler baseHandler;
   private HiveClientPool clientPool;
   private final String hiveWarehousePath;
 
@@ -230,7 +230,7 @@ private TServer newThriftServer(TServerSocket socket, int poolSize, HiveConf con
       throws Exception {
     HiveConf serverConf = new HiveConf(conf);
     serverConf.set(
-        HiveConf.ConfVars.METASTORECONNECTURLKEY.varname,
+        HiveConf.ConfVars.METASTORE_CONNECT_URL_KEY.varname,
         "jdbc:derby:" + DERBY_PATH + ";create=true");
     baseHandler = HMS_HANDLER_CTOR.newInstance("new db based metaserver", serverConf);
     IHMSHandler handler = GET_BASE_HMS_HANDLER.invoke(serverConf, baseHandler, false);
@@ -247,8 +247,8 @@ private TServer newThriftServer(TServerSocket socket, int poolSize, HiveConf con
   }
 
   private void initConf(HiveConf conf, int port) {
-    conf.set(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://localhost:" + port);
-    conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, hiveWarehousePath);
+    conf.set(HiveConf.ConfVars.METASTORE_URIS.varname, "thrift://localhost:" + port);
+    conf.set(HiveConf.ConfVars.METASTORE_WAREHOUSE.varname, hiveWarehousePath);
     conf.set(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL.varname, "false");
     conf.set(HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES.varname, "false");
     conf.set("iceberg.hive.client-pool-size", "2");