支持hdfs配置UseDatanodeHostname (PaddlePaddle#1338)

D0m021ng · Apr 10, 2024 · 9828d8f · 9828d8f
1 parent 0038a39
commit 9828d8f
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 4 deletions.
diff --git a/pkg/fs/client/ufs/hdfs.go b/pkg/fs/client/ufs/hdfs.go
@@ -636,6 +636,10 @@ func NewHdfsFileSystem(properties map[string]interface{}) (UnderFileStorage, err
 	options := hdfs.ClientOptions{
 		Addresses: strings.Split(nameNodeAddress, ","),
 	}
+	useDatanodeHostname, ok := properties[common.UseDatanodeHostname].(bool)
+	if ok && useDatanodeHostname {
+		options.UseDatanodeHostname = true
+	}
 	options.User = properties[common.UserKey].(string)
 	cli, err := hdfs.NewClient(options)
 	if err != nil {

diff --git a/pkg/fs/client/ufs/hdfs_kerberos.go b/pkg/fs/client/ufs/hdfs_kerberos.go
@@ -123,6 +123,10 @@ func NewHdfsWithKerberosFileSystem(properties map[string]interface{}) (UnderFile
 	options := hdfs.ClientOptions{
 		Addresses: strings.Split(nameNodeAddress, ","),
 	}
+	useDatanodeHostname, ok := properties[common.UseDatanodeHostname].(bool)
+	if ok && useDatanodeHostname {
+		options.UseDatanodeHostname = true
+	}
 	krbConfig, _ := buildKerberosConf(properties)
 	if krbClient, err := NewKerberosClientWithKeyTab(krbConfig); err == nil {
 		options.KerberosClient = krbClient

diff --git a/pkg/fs/common/common.go b/pkg/fs/common/common.go
@@ -38,10 +38,11 @@ const (
 	Type    = "type"
 
 	// HDFS properties
-	NameNodeAddress = "dfs.namenode.address"
-	UserKey         = "user"
-	BlockSizeKey    = "blockSize"
-	ReplicationKey  = "replication"
+	NameNodeAddress     = "dfs.namenode.address"
+	UseDatanodeHostname = "dfs.client.use.datanode.hostname"
+	UserKey             = "user"
+	BlockSizeKey        = "blockSize"
+	ReplicationKey      = "replication"
 
 	Sts             = "sts"
 	Token           = "token"