Skip to content

Commit 720a4d6

Browse files
ryukobayashiryukobayashi
committed
Merge branch 'master' into HIVE-27370 
2 parents 8127a2b + 8ff73bb commit 720a4d6

File tree

23 files changed

+347
-287
lines changed

23 files changed

+347
-287
lines changed

‎Jenkinsfile

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,8 @@ def hdbPodTemplate(closure) {
135135
resourceLimitCpu: '8000m',
136136
resourceRequestMemory: '6400Mi',
137137
resourceLimitMemory: '12000Mi',
138+
resourceRequestEphemeralStorage: '10Gi',
139+
resourceLimitEphemeralStorage: '20Gi',
138140
envVars: [
139141
envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2376'),
140142
envVar(key: 'DOCKER_TLS_VERIFY', value: '1'),
@@ -226,7 +228,7 @@ jobWrappers {
226228
$class: 'GitSCM',
227229
branches: scm.branches,
228230
doGenerateSubmoduleConfigurations: scm.doGenerateSubmoduleConfigurations,
229-
extensions: scm.extensions,
231+
extensions: [ cloneOption(honorRefspec: true, depth: 50, noTags: true, shallow: true) ],
230232
userRemoteConfigs: scm.userRemoteConfigs + [[
231233
name: 'origin',
232234
refspec: scm.userRemoteConfigs[0].refspec+ " +refs/heads/${CHANGE_TARGET}:refs/remotes/origin/target",

‎itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,8 @@ public void testListener() throws Exception {
240240

241241

242242
driver.run(String.format("alter table %s rename to %s", tblName, renamed));
243+
// remove the last auth call, which is for the get_partitions_req
244+
authCalls.remove(authCalls.size() - 1);
243245
listSize = authCalls.size();
244246

245247
Table renamedTableFromEvent = (

‎ql/src/java/org/apache/hadoop/hive/ql/queryhistory/repository/IcebergRepository.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ protected Table createTable(Hive hive, Database db) throws HiveException {
8989
ICEBERG_STORAGE_HANDLER);
9090
table.setProperty("table_type", "ICEBERG");
9191
table.setProperty("write.format.default", "orc");
92+
// set the default max snapshot age to 1 day for query history
93+
// this is applied only during table creation, it can be manually altered afterward.
94+
table.setProperty("history.expire.max-snapshot-age-ms", Integer.toString(24 * 60 * 60 * 1000));
9295
table.setProperty(hive_metastoreConstants.META_TABLE_NAME, QUERY_HISTORY_DB_TABLE_NAME);
9396

9497
table.setFields(schema.getFields());

‎standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1853,26 +1853,26 @@ public enum ConfVars {
18531853
new StringSetValidator("simple", "jwt"),
18541854
"Property-maps servlet authentication method (simple or jwt)."
18551855
),
1856-
ICEBERG_CATALOG_SERVLET_FACTORY("metastore.iceberg.catalog.servlet.factory",
1857-
"hive.metastore.iceberg.catalog.servlet.factory",
1858-
"org.apache.iceberg.rest.HMSCatalogFactory",
1859-
"HMS Iceberg Catalog servlet factory class name."
1856+
CATALOG_SERVLET_FACTORY("metastore.catalog.servlet.factory",
1857+
"hive.metastore.catalog.servlet.factory",
1858+
"org.apache.iceberg.rest.HMSCatalogFactory",
1859+
"HMS Catalog servlet factory class name. The default serves Iceberg REST API."
18601860
+ "The factory needs to expose a method: "
18611861
+ "public static HttpServlet createServlet(Configuration configuration);"
18621862
),
1863-
ICEBERG_CATALOG_SERVLET_PATH("metastore.iceberg.catalog.servlet.path",
1864-
"hive.metastore.iceberg.catalog.servlet.path", "iceberg",
1865-
"HMS Iceberg Catalog servlet path component of URL endpoint."
1866-
),
1867-
ICEBERG_CATALOG_SERVLET_PORT("metastore.iceberg.catalog.servlet.port",
1868-
"hive.metastore.iceberg.catalog.servlet.port", -1,
1869-
"HMS Iceberg Catalog servlet server port. Negative value disables the servlet," +
1863+
CATALOG_SERVLET_PORT("metastore.catalog.servlet.port",
1864+
"hive.metastore.catalog.servlet.port", -1,
1865+
"HMS Catalog servlet server port. Negative value disables the servlet," +
18701866
" 0 will let the system determine the catalog server port," +
18711867
" positive value will be used as-is."
18721868
),
1873-
ICEBERG_CATALOG_SERVLET_AUTH("metastore.iceberg.catalog.servlet.auth",
1874-
"hive.metastore.iceberg.catalog.servlet.auth", "jwt", new StringSetValidator("simple", "jwt"),
1875-
"HMS Iceberg Catalog servlet authentication method (simple or jwt)."
1869+
CATALOG_SERVLET_AUTH("metastore.catalog.servlet.auth",
1870+
"hive.metastore.catalog.servlet.auth", "jwt", new StringSetValidator("none", "simple", "jwt"),
1871+
"HMS Catalog servlet authentication method (none, simple, or jwt)."
1872+
),
1873+
ICEBERG_CATALOG_SERVLET_PATH("metastore.iceberg.catalog.servlet.path",
1874+
"hive.metastore.iceberg.catalog.servlet.path", "iceberg",
1875+
"HMS Iceberg Catalog servlet path component of URL endpoint."
18761876
),
18771877
ICEBERG_CATALOG_CACHE_EXPIRY("metastore.iceberg.catalog.cache.expiry",
18781878
"hive.metastore.iceberg.catalog.cache.expiry", -1,

‎standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import javax.servlet.http.HttpServlet;
2424
import org.apache.hadoop.conf.Configuration;
2525
import org.apache.hadoop.hive.metastore.ServletSecurity;
26+
import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
2627
import org.apache.hadoop.hive.metastore.ServletServerBuilder;
2728
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
2829
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
@@ -47,7 +48,7 @@ public class HMSCatalogFactory {
4748
* @param conf the configuration
4849
*/
4950
private HMSCatalogFactory(Configuration conf) {
50-
port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT);
51+
port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.CATALOG_SERVLET_PORT);
5152
path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH);
5253
this.configuration = conf;
5354
}
@@ -98,8 +99,8 @@ private Catalog createCatalog() {
9899
* @return the servlet
99100
*/
100101
private HttpServlet createServlet(Catalog catalog) {
101-
String authType = MetastoreConf.getVar(configuration, ConfVars.ICEBERG_CATALOG_SERVLET_AUTH);
102-
ServletSecurity security = new ServletSecurity(authType, configuration);
102+
String authType = MetastoreConf.getVar(configuration, ConfVars.CATALOG_SERVLET_AUTH);
103+
ServletSecurity security = new ServletSecurity(AuthType.fromString(authType), configuration);
103104
return security.proxy(new HMSCatalogServlet(new HMSCatalogAdapter(catalog)));
104105
}
105106

@@ -116,7 +117,7 @@ private HttpServlet createServlet() {
116117

117118
/**
118119
* Factory method to describe Iceberg servlet.
119-
* <p>This method name is found through configuration as {@link MetastoreConf.ConfVars#ICEBERG_CATALOG_SERVLET_FACTORY}
120+
* <p>This method name is found through configuration as {@link MetastoreConf.ConfVars#CATALOG_SERVLET_FACTORY}
120121
* and looked up through reflection to start from HMS.</p>
121122
*
122123
* @param configuration the configuration

‎standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogJwtAuth.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.apache.iceberg.rest;
2121

2222
import java.util.Map;
23+
import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
2324
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
2425
import org.apache.iceberg.exceptions.NotAuthorizedException;
2526
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
@@ -32,8 +33,8 @@
3233
@Category(MetastoreCheckinTest.class)
3334
class TestRESTCatalogJwtAuth extends BaseRESTCatalogTests {
3435
@RegisterExtension
35-
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION = HiveRESTCatalogServerExtension.builder()
36-
.jwt().build();
36+
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
37+
HiveRESTCatalogServerExtension.builder(AuthType.JWT).build();
3738

3839
@Override
3940
protected Map<String, String> getDefaultClientConfiguration() throws Exception {

‎standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogNoneAuth.java

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iceberg.rest;
21+
22+
import java.util.Map;
23+
import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
24+
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
25+
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
26+
import org.junit.experimental.categories.Category;
27+
import org.junit.jupiter.api.extension.RegisterExtension;
28+
29+
@Category(MetastoreCheckinTest.class)
30+
class TestRESTCatalogNoneAuth extends BaseRESTCatalogTests {
31+
@RegisterExtension
32+
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
33+
HiveRESTCatalogServerExtension.builder(AuthType.NONE).build();
34+
35+
@Override
36+
protected Map<String, String> getDefaultClientConfiguration() {
37+
return Map.of(
38+
"uri", REST_CATALOG_EXTENSION.getRestEndpoint()
39+
);
40+
}
41+
}

‎standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestRESTCatalogSimpleAuth.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.apache.iceberg.rest;
2121

2222
import java.util.Map;
23+
import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
2324
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
2425
import org.apache.iceberg.exceptions.NotAuthorizedException;
2526
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
@@ -31,8 +32,8 @@
3132
@Category(MetastoreCheckinTest.class)
3233
class TestRESTCatalogSimpleAuth extends BaseRESTCatalogTests {
3334
@RegisterExtension
34-
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION = HiveRESTCatalogServerExtension.builder()
35-
.build();
35+
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
36+
HiveRESTCatalogServerExtension.builder(AuthType.SIMPLE).build();
3637

3738
@Override
3839
protected Map<String, String> getDefaultClientConfiguration() {

‎standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/HiveRESTCatalogServerExtension.java

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.stream.Stream;
2626
import org.apache.commons.io.FileUtils;
2727
import org.apache.hadoop.conf.Configuration;
28+
import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
2829
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
2930
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
3031
import org.junit.jupiter.api.extension.AfterAllCallback;
@@ -41,16 +42,16 @@ public class HiveRESTCatalogServerExtension implements BeforeAllCallback, Before
4142
private final JwksServer jwksServer;
4243
private final RESTCatalogServer restCatalogServer;
4344

44-
public HiveRESTCatalogServerExtension(boolean jwtEnabled) {
45+
private HiveRESTCatalogServerExtension(AuthType authType) {
4546
this.conf = MetastoreConf.newMetastoreConf();
46-
if (jwtEnabled) {
47+
MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH, authType.name());
48+
if (authType == AuthType.JWT) {
4749
jwksServer = new JwksServer();
48-
MetastoreConf.setVar(conf, ConfVars.ICEBERG_CATALOG_SERVLET_AUTH, "jwt");
50+
MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH, "jwt");
4951
MetastoreConf.setVar(conf, ConfVars.THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL,
5052
String.format("http://localhost:%d/jwks", jwksServer.getPort()));
5153
} else {
5254
jwksServer = null;
53-
MetastoreConf.setVar(conf, ConfVars.ICEBERG_CATALOG_SERVLET_AUTH, "simple");
5455
}
5556
restCatalogServer = new RESTCatalogServer();
5657
}
@@ -96,22 +97,18 @@ public String getRestEndpoint() {
9697
}
9798

9899
public static class Builder {
99-
private boolean jwtEnabled = false;
100+
private final AuthType authType;
100101

101-
private Builder() {
102-
}
103-
104-
public Builder jwt() {
105-
jwtEnabled = true;
106-
return this;
102+
private Builder(AuthType authType) {
103+
this.authType = authType;
107104
}
108105

109106
public HiveRESTCatalogServerExtension build() {
110-
return new HiveRESTCatalogServerExtension(jwtEnabled);
107+
return new HiveRESTCatalogServerExtension(authType);
111108
}
112109
}
113110

114-
public static Builder builder() {
115-
return new Builder();
111+
public static Builder builder(AuthType authType) {
112+
return new Builder(authType);
116113
}
117114
}

‎standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/extension/RESTCatalogServer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public class RESTCatalogServer {
3838
private int restPort = -1;
3939

4040
private static int createMetastoreServerWithRESTCatalog(int restPort, Configuration conf) throws Exception {
41-
MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT, restPort);
41+
MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.CATALOG_SERVLET_PORT, restPort);
4242
return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf,
4343
true, false, false, false);
4444
}

‎standalone-metastore/metastore-server/src/docker/conf/metastore-site.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,11 @@
3333
<value>false</value>
3434
</property>
3535
<property>
36-
<name>hive.metastore.iceberg.catalog.servlet.port</name>
36+
<name>metastore.catalog.servlet.port</name>
3737
<value>9001</value>
3838
</property>
3939
<property>
40-
<name>hive.metastore.iceberg.catalog.servlet.auth</name>
41-
<value>simple</value>
40+
<name>metastore.catalog.servlet.auth</name>
41+
<value>none</value>
4242
</property>
4343
</configuration>

‎standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java

Lines changed: 8 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.hadoop.hive.metastore;
2020

2121
import static org.apache.commons.lang3.StringUtils.repeat;
22+
import static org.apache.hadoop.hive.metastore.MetastoreDirectSqlUtils.getModelIdentity;
2223

2324
import java.util.ArrayList;
2425
import java.util.Collections;
@@ -41,11 +42,6 @@
4142
import org.apache.hadoop.hive.metastore.model.MSerDeInfo;
4243
import org.apache.hadoop.hive.metastore.model.MStorageDescriptor;
4344
import org.apache.hadoop.hive.metastore.model.MStringList;
44-
import org.datanucleus.ExecutionContext;
45-
import org.datanucleus.api.jdo.JDOPersistenceManager;
46-
import org.datanucleus.metadata.AbstractClassMetaData;
47-
import org.datanucleus.metadata.AbstractMemberMetaData;
48-
import org.datanucleus.metadata.IdentityType;
4945

5046
/**
5147
* This class contains the methods to insert into tables on the underlying database using direct SQL
@@ -68,16 +64,6 @@ interface BatchExecutionContext {
6864
void execute(String batchQueryText, int batchRowCount) throws MetaException;
6965
}
7066

71-
private Long getDataStoreId(Class<?> modelClass) throws MetaException {
72-
ExecutionContext ec = ((JDOPersistenceManager) pm).getExecutionContext();
73-
AbstractClassMetaData cmd = ec.getMetaDataManager().getMetaDataForClass(modelClass, ec.getClassLoaderResolver());
74-
if (cmd.getIdentityType() == IdentityType.DATASTORE) {
75-
return (Long) ec.getStoreManager().getValueGenerationStrategyValue(ec, cmd, null);
76-
} else {
77-
throw new MetaException("Identity type is not datastore.");
78-
}
79-
}
80-
8167
private void insertInBatch(String tableName, String columns, int columnCount, int rowCount,
8268
BatchExecutionContext batchExecutionContext) throws MetaException {
8369
if (rowCount == 0 || columnCount == 0) {
@@ -751,24 +737,24 @@ public void addPartitions(List<MPartition> parts, List<List<MPartitionPrivilege>
751737
|| sd.getCD().getCols() == null) {
752738
throw new MetaException("Invalid partition");
753739
}
754-
Long serDeId = getDataStoreId(MSerDeInfo.class);
740+
Long serDeId = getModelIdentity(pm, MSerDeInfo.class);
755741
serdeIdToSerDeInfo.put(serDeId, sd.getSerDeInfo());
756742

757743
Long cdId;
758744
LongIdentity storeId = (LongIdentity) pm.getObjectId(sd.getCD());
759745
if (storeId == null) {
760-
cdId = getDataStoreId(MColumnDescriptor.class);
746+
cdId = getModelIdentity(pm, MColumnDescriptor.class);
761747
cdIdToColumnDescriptor.put(cdId, sd.getCD());
762748
} else {
763749
cdId = (Long) storeId.getKeyAsObject();
764750
}
765751

766-
Long sdId = getDataStoreId(MStorageDescriptor.class);
752+
Long sdId = getModelIdentity(pm, MStorageDescriptor.class);
767753
sdIdToStorageDescriptor.put(sdId, sd);
768754
sdIdToSerdeId.put(sdId, serDeId);
769755
sdIdToCdId.put(sdId, cdId);
770756

771-
Long partId = getDataStoreId(MPartition.class);
757+
Long partId = getModelIdentity(pm, MPartition.class);
772758
partIdToPartition.put(partId, part);
773759
partIdToSdId.put(partId, sdId);
774760

@@ -781,7 +767,7 @@ public void addPartitions(List<MPartition> parts, List<List<MPartitionPrivilege>
781767
if (CollectionUtils.isNotEmpty(sd.getSkewedColValues())) {
782768
int skewedValCount = sd.getSkewedColValues().size();
783769
for (int i = 0; i < skewedValCount; i++) {
784-
Long stringListId = getDataStoreId(MStringList.class);
770+
Long stringListId = getModelIdentity(pm, MStringList.class);
785771
stringListIds.add(stringListId);
786772
stringListIdToSdId.put(stringListId, sdId);
787773
List<String> stringList = sd.getSkewedColValues().get(i).getInternalList();
@@ -795,13 +781,13 @@ public void addPartitions(List<MPartition> parts, List<List<MPartitionPrivilege>
795781

796782
List<MPartitionPrivilege> partPrivileges = partPrivilegesList.get(index);
797783
for (MPartitionPrivilege partPrivilege : partPrivileges) {
798-
Long partGrantId = getDataStoreId(MPartitionPrivilege.class);
784+
Long partGrantId = getModelIdentity(pm, MPartitionPrivilege.class);
799785
partGrantIdToPrivilege.put(partGrantId, partPrivilege);
800786
partGrantIdToPartId.put(partGrantId, partId);
801787
}
802788
List<MPartitionColumnPrivilege> partColPrivileges = partColPrivilegesList.get(index);
803789
for (MPartitionColumnPrivilege partColPrivilege : partColPrivileges) {
804-
Long partColumnGrantId = getDataStoreId(MPartitionColumnPrivilege.class);
790+
Long partColumnGrantId = getModelIdentity(pm, MPartitionColumnPrivilege.class);
805791
partColumnGrantIdToPrivilege.put(partColumnGrantId, partColPrivilege);
806792
partColumnGrantIdToPartId.put(partColumnGrantId, partId);
807793
}

0 commit comments

Comments
 (0)