HIVE-28655: Implement HMS Related Drop Stats Changes, Reset COLUMN_STAT_ACCURATE After Dropping

Author: Hongdan Zhu

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java

@@ -7491,6 +7491,8 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
               .collect(Collectors.toList()) : colNames) {
             for (String partName : partNames) {
               List<String> partVals = getPartValsFromName(table, partName);
+              Partition partition = rawStore.getPartition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, partVals);
+              Map<String, String> partParams = partition.getParameters();
               if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
                 MetaStoreListenerNotifier.notifyEvent(transactionalListeners, eventType,
                     new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
@@ -7502,6 +7504,15 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
           }
         }
       }
+      // on the table level, partially delete(update) table level parameter COLUMN_STATS_ACCURATE
+      if (colNames == null || colNames.isEmpty()){
+        // remove all column names in parameter COLUMN_STATS_ACCURATE
+        StatsSetupConst.clearColumnStatsState(table.getParameters());
+      } else {
+        // remove the deleted column names in parameter COLUMN_STATS_ACCURATE
+        StatsSetupConst.removeColumnStatsState(table.getParameters(), colNames);
+      }
+      rawStore.alterTable(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, table, null);
       committed = rawStore.commitTransaction();
     } finally {
       if (!committed) {

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java

@@ -183,7 +183,7 @@ public static <T> String getIdListForIn(Collection<T> objectIds) throws MetaExce
 
   // Table names with schema name, if necessary
   @TableName
-  private String DBS, TBLS, PARTITIONS, DATABASE_PARAMS, PARTITION_PARAMS, SORT_COLS, SD_PARAMS,
+  private String DBS, TBLS, PARTITIONS, DATABASE_PARAMS, TABLE_PARAMS, PARTITION_PARAMS, SORT_COLS, SD_PARAMS,
       SDS, SERDES, SKEWED_STRING_LIST_VALUES, SKEWED_VALUES, BUCKETING_COLS, SKEWED_COL_NAMES,
       SKEWED_COL_VALUE_LOC_MAP, COLUMNS_V2, PARTITION_KEYS, SERDE_PARAMS, PART_COL_STATS, KEY_CONSTRAINTS,
       TAB_COL_STATS, PARTITION_KEY_VALS, PART_PRIVS, PART_COL_PRIVS, SKEWED_STRING_LIST, CDS,

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java

@@ -10260,6 +10260,8 @@ public boolean deletePartitionColumnStatistics(String catName, String dbName, St
     }
     dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME);
     catName = normalizeIdentifier(catName);
+    // use directSql to compeletely delete the parameter COLUMN_STATS_ACCUARTE on partition level
+    directSql.deleteColumnStatsState(getTable(catName, dbName, tableName).getId());
     return new GetHelper<Boolean>(catName, dbName, tableName, true, true) {
       @Override
       protected String describeResult() {

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java

@@ -1846,6 +1846,11 @@ public void testColumnStatistics() throws Throwable {
       List<ColumnStatisticsObj> stats = client.getTableColumnStatistics(
           dbName, tblName, Lists.newArrayList(colName[1]), ENGINE);
       assertTrue("stats are not empty: " + stats, stats.isEmpty());
+      // test if all columns are deleted from parameter COLUMN_STATS_ACCURATE
+      Map<String, String> tableParams = client.getTable(dbName, tblName).getParameters();
+      String table_column_stats_accurate = tableParams.get("COLUMN_STATS_ACCURATE");
+      assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + tblName, table_column_stats_accurate == null ||
+              (!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
 
       colStats.setStatsDesc(statsDesc);
       colStats.setStatsObj(statsObjs);
@@ -1863,6 +1868,11 @@ public void testColumnStatistics() throws Throwable {
       // multiple columns
       request.setCol_names(Arrays.asList(colName));
       assertTrue(client.deleteColumnStatistics(request));
+      // test if the columns in colName array are deleted from parameter COLUMN_STATS_ACCURATE
+      tableParams = client.getTable(dbName, tblName).getParameters();
+      table_column_stats_accurate = tableParams.get("COLUMN_STATS_ACCURATE");
+      assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + tblName, table_column_stats_accurate == null ||
+              (!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
       colStats3 = client.getTableColumnStatistics(
           dbName, tblName, Lists.newArrayList(colName), ENGINE);
       assertTrue("stats are not empty: " + colStats3, colStats3.isEmpty());
@@ -1958,6 +1968,12 @@ public void testColumnStatistics() throws Throwable {
           Lists.newArrayList(partitions.get(0), partitions.get(1), partitions.get(2)), Lists.newArrayList(colName), ENGINE);
      assertEquals(1, stats2.size());
      assertEquals(2, stats2.get(partitions.get(2)).size());
+     // test if all columns are deleted from parameter COLUMN_STATS_ACCURATE
+     Partition partition_0 = client.getPartition(dbName, tblName, partitions.get(0));
+     Map<String, String> partitionParams = partition_0.getParameters();
+     String partition_column_stats_accurate = partitionParams.get("COLUMN_STATS_ACCURATE");
+     assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + partitions.get(0),partition_column_stats_accurate == null ||
+             (!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
 
      // no partition or column name is set
      request.unsetPart_names();