HIVE-28655: Implement HMS Related Drop Stats Changes, Reset COLUMN_STAT_ACCURATE After Dropping

Author: Hongdan Zhu

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java

@@ -7440,6 +7440,13 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
             }
             events.add(new DeleteTableColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, colName, engine, this));
           }
+          if (colNames == null){
+            // remove all column names in parameter COLUMN_STATS_ACCURATE
+            StatsSetupConst.clearColumnStatsState(table.getParameters());
+          } else {
+            // remove the deleted column names in parameter COLUMN_STATS_ACCURATE
+            StatsSetupConst.removeColumnStatsState(table.getParameters(), colNames);
+          }
         }
       } else {
         List<String> partNames = new ArrayList<>();
@@ -7460,13 +7467,22 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
               .collect(Collectors.toList()) : colNames) {
             for (String partName : partNames) {
               List<String> partVals = getPartValsFromName(table, partName);
+              Partition partition = rawStore.getPartition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, partVals);
+              Map<String, String> partParams = partition.getParameters();
               if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
                 MetaStoreListenerNotifier.notifyEvent(transactionalListeners, eventType,
                     new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
                         partName, partVals, colName, engine, this));
               }
               events.add(new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
                   partName, partVals, colName, engine, this));
+              if (colNames == null) {
+                // remove all column names in parameter COLUMN_STATS_ACCURATE
+                StatsSetupConst.clearColumnStatsState(partParams);
+              } else {
+                // remove the deleted column names in parameter COLUMN_STATS_ACCURATE
+                StatsSetupConst.removeColumnStatsState(partParams, colNames);
+              }
             }
           }
         }

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java

@@ -1846,6 +1846,11 @@ public void testColumnStatistics() throws Throwable {
       List<ColumnStatisticsObj> stats = client.getTableColumnStatistics(
           dbName, tblName, Lists.newArrayList(colName[1]), ENGINE);
       assertTrue("stats are not empty: " + stats, stats.isEmpty());
+      // test if all columns are deleted from parameter COLUMN_STATS_ACCURATE
+      Map<String, String> tableParams = client.getTable(dbName, tblName).getParameters();
+      String table_column_stats_accurate = tableParams.get("COLUMN_STATS_ACCURATE");
+      assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + tblName, table_column_stats_accurate == null ||
+              (!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
 
       colStats.setStatsDesc(statsDesc);
       colStats.setStatsObj(statsObjs);
@@ -1863,6 +1868,11 @@ public void testColumnStatistics() throws Throwable {
       // multiple columns
       request.setCol_names(Arrays.asList(colName));
       assertTrue(client.deleteColumnStatistics(request));
+      // test if the columns in colName array are deleted from parameter COLUMN_STATS_ACCURATE
+      tableParams = client.getTable(dbName, tblName).getParameters();
+      table_column_stats_accurate = tableParams.get("COLUMN_STATS_ACCURATE");
+      assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + tblName, table_column_stats_accurate == null ||
+              (!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
       colStats3 = client.getTableColumnStatistics(
           dbName, tblName, Lists.newArrayList(colName), ENGINE);
       assertTrue("stats are not empty: " + colStats3, colStats3.isEmpty());
@@ -1958,6 +1968,12 @@ public void testColumnStatistics() throws Throwable {
           Lists.newArrayList(partitions.get(0), partitions.get(1), partitions.get(2)), Lists.newArrayList(colName), ENGINE);
      assertEquals(1, stats2.size());
      assertEquals(2, stats2.get(partitions.get(2)).size());
+     // test if all columns are deleted from parameter COLUMN_STATS_ACCURATE
+     Partition partition_0 = client.getPartition(dbName, tblName, partitions.get(0));
+     Map<String, String> partitionParams = partition_0.getParameters();
+     String partition_column_stats_accurate = partitionParams.get("COLUMN_STATS_ACCURATE");
+     assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + partitions.get(0),partition_column_stats_accurate == null ||
+             (!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
 
      // no partition or column name is set
      request.unsetPart_names();