001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.commons.vfs2.provider.hdfs; 018 019import java.io.InputStream; 020import java.net.MalformedURLException; 021import java.net.URL; 022import java.util.Arrays; 023import java.util.stream.Stream; 024 025import org.apache.commons.lang3.StringUtils; 026import org.apache.commons.vfs2.FileSystem; 027import org.apache.commons.vfs2.FileSystemConfigBuilder; 028import org.apache.commons.vfs2.FileSystemOptions; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.Path; 031 032/** 033 * Configuration settings for the HdfsFileSystem. 034 * 035 * @since 2.1 036 */ 037public final class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder { 038 039 private static final HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder(); 040 private static final String KEY_CONFIG_NAMES = "configNames"; 041 private static final String KEY_CONFIG_PATHS = "configPaths"; 042 private static final String KEY_CONFIG_URLS = "configURLs"; 043 private static final String KEY_CONFIG_STREAM = "configStream"; 044 private static final String KEY_CONFIG_CONF = "configConf"; 045 046 /** 047 * Gets the singleton instance. 048 * 049 * @return HdfsFileSystemConfigBuilder instance 050 */ 051 public static HdfsFileSystemConfigBuilder getInstance() { 052 return BUILDER; 053 } 054 055 /** 056 * Constructs a new instance. 057 */ 058 private HdfsFileSystemConfigBuilder() { 059 super("hdfs."); 060 } 061 062 /** 063 * @return HDFSFileSystem 064 */ 065 @Override 066 protected Class<? extends FileSystem> getConfigClass() { 067 return HdfsFileSystem.class; 068 } 069 070 /** 071 * Gets the alternate configuration object. 072 * 073 * @return alternate configuration object or {@code null}. 074 * @param opts The FileSystemOptions. 075 * @see #setConfigConfiguration(FileSystemOptions, Configuration) 076 */ 077 public Configuration getConfigConfiguration(final FileSystemOptions opts) { 078 return getParam(opts, KEY_CONFIG_CONF); 079 } 080 081 /** 082 * Gets the alternate configuration input stream. 083 * 084 * @return alternate configuration input stream or {@code null}. 085 * @param opts The FileSystemOptions. 086 * @see #setConfigInputStream(FileSystemOptions, InputStream) 087 */ 088 public InputStream getConfigInputStream(final FileSystemOptions opts) { 089 return getParam(opts, KEY_CONFIG_STREAM); 090 } 091 092 /** 093 * Gets the names of alternate configuration resources. 094 * 095 * @return resource name list of alternate configurations or {@code null}. 096 * @param opts The FileSystemOptions. 097 * @see #setConfigName(FileSystemOptions, String) 098 */ 099 public String[] getConfigNames(final FileSystemOptions opts) { 100 final String names = this.getString(opts, KEY_CONFIG_NAMES); 101 return StringUtils.isEmpty(names) ? null : names.split(","); 102 } 103 104 /** 105 * Gets the paths of alternate configuration file system files. 106 * 107 * @return list of full paths of alternate configuration files or {@code null}. 108 * @param opts The FileSystemOptions. 109 * @see #setConfigPath(FileSystemOptions, Path) 110 */ 111 public Path[] getConfigPaths(final FileSystemOptions opts) { 112 final String pathNames = this.getString(opts, KEY_CONFIG_PATHS); 113 if (StringUtils.isEmpty(pathNames)) { 114 return null; 115 } 116 return Stream.of(pathNames.split(",")).map(Path::new).toArray(Path[]::new); 117 } 118 119 /** 120 * Gets URLs of alternate configurations. 121 * 122 * @return list of alternate configuration URLs or {@code null}. 123 * @param opts The FileSystemOptions. 124 * @see #setConfigURL(FileSystemOptions, URL) 125 */ 126 public URL[] getConfigURLs(final FileSystemOptions opts) { 127 final String urlNames = this.getString(opts, KEY_CONFIG_URLS); 128 if (StringUtils.isEmpty(urlNames)) { 129 return null; 130 } 131 final String[] urls = urlNames.split(","); 132 final URL[] realURLs = new URL[urls.length]; 133 Arrays.setAll(realURLs, i -> { 134 try { 135 return new URL(urls[i]); 136 } catch (final MalformedURLException e) { 137 // This should never happen because we save it in the proper form. 138 throw new IllegalArgumentException(urls[i], e); 139 } 140 141 }); 142 return realURLs; 143 } 144 145 /** 146 * Sets the configuration object to be loaded after the defaults. 147 * <p> 148 * Specifies an already initialized configuration object to override any specific HDFS settings. The property will 149 * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Configuration)} after the URL was set as 150 * the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}. 151 * </p> 152 * <p> 153 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order 154 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses 155 * too. 156 * </p> 157 * 158 * @param opts The FileSystemOptions to modify. 159 * @param configuration additional configuration object or {@code null} to unset any configuration object previously 160 * set. 161 */ 162 public void setConfigConfiguration(final FileSystemOptions opts, final Configuration configuration) { 163 this.setParam(opts, KEY_CONFIG_CONF, configuration); 164 } 165 166 /** 167 * Sets the input stream of configuration file to be loaded after the defaults. 168 * <p> 169 * Specifies an input stream connected to a config file to override any specific HDFS settings. The property will be 170 * passed on to {@code org.apache.hadoop.conf.Configuration#addResource(InputStream)} after the URL was set as the 171 * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}. 172 * </p> 173 * <p> 174 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order 175 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses 176 * too. 177 * </p> 178 * 179 * @param opts The FileSystemOptions to modify. 180 * @param inputStream input stream of additional configuration file or {@code null} to unset the configuration input 181 * stream previously set up. 182 */ 183 public void setConfigInputStream(final FileSystemOptions opts, final InputStream inputStream) { 184 this.setParam(opts, KEY_CONFIG_STREAM, inputStream); 185 } 186 187 /** 188 * Sets the name of configuration resource to be loaded after the defaults. 189 * <p> 190 * Specifies the name of a config resource to override any specific HDFS settings. The property will be passed on to 191 * {@code org.apache.hadoop.conf.Configuration#addResource(String)} after the URL was set as the default name with: 192 * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}. 193 * </p> 194 * <p> 195 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order 196 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses 197 * too. 198 * </p> 199 * <p> 200 * This method may be called multiple times and all the specified resources will be loaded in the order they were 201 * specified. 202 * </p> 203 * <p> 204 * Note also, that if a list of names is provided, separated by commas ({@code ","}), that this will work the same 205 * as calling this method a number of times with just one name each. 206 * </p> 207 * 208 * @param opts The FileSystemOptions to modify. 209 * @param name resource name of additional configuration or {@code null} to unset all the values set so far. 210 * @see #getConfigNames 211 */ 212 public void setConfigName(final FileSystemOptions opts, final String name) { 213 if (StringUtils.isEmpty(name)) { 214 this.setParam(opts, KEY_CONFIG_NAMES, null); 215 } else { 216 final String previousNames = this.getString(opts, KEY_CONFIG_NAMES); 217 if (previousNames == null || previousNames.isEmpty()) { 218 this.setParam(opts, KEY_CONFIG_NAMES, name); 219 } else { 220 this.setParam(opts, KEY_CONFIG_NAMES, previousNames + "," + name); 221 } 222 } 223 } 224 225 /** 226 * Sets the full path of configuration file to be loaded after the defaults. 227 * <p> 228 * Specifies the path of a local file system config file to override any specific HDFS settings. The property will 229 * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Path)} after the URL was set as the 230 * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}. 231 * </p> 232 * <p> 233 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order 234 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses 235 * too. 236 * </p> 237 * <p> 238 * This method may be called multiple times and all the specified resources will be loaded in the order they were 239 * specified. 240 * </p> 241 * 242 * @param opts The FileSystemOptions to modify. 243 * @param path full path of additional configuration file (local file system) or {@code null} to unset all the path 244 * values set so far. 245 */ 246 public void setConfigPath(final FileSystemOptions opts, final Path path) { 247 if (path == null) { 248 this.setParam(opts, KEY_CONFIG_PATHS, null); 249 } else { 250 final String previousPathNames = this.getString(opts, KEY_CONFIG_PATHS); 251 if (StringUtils.isEmpty(previousPathNames)) { 252 this.setParam(opts, KEY_CONFIG_PATHS, path.toString()); 253 } else { 254 this.setParam(opts, KEY_CONFIG_PATHS, previousPathNames + "," + path); 255 } 256 } 257 } 258 259 /** 260 * Sets the URL of configuration file to be loaded after the defaults. 261 * <p> 262 * Specifies the URL of a config file to override any specific HDFS settings. The property will be passed on to 263 * {@code org.apache.hadoop.conf.Configuration#addResource(URL)} after the URL was set as the default name with: 264 * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}. 265 * </p> 266 * <p> 267 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order 268 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses 269 * too. 270 * </p> 271 * <p> 272 * This method may be called multiple times and all the specified resources will be loaded in the order they were 273 * specified. 274 * </p> 275 * 276 * @param opts The FileSystemOptions to modify. 277 * @param url URL of additional configuration file or {@code null} to unset all the URL values set so far. 278 */ 279 public void setConfigURL(final FileSystemOptions opts, final URL url) { 280 if (url == null) { 281 this.setParam(opts, KEY_CONFIG_URLS, null); 282 } else { 283 final String previousURLNames = this.getString(opts, KEY_CONFIG_URLS); 284 if (StringUtils.isEmpty(previousURLNames)) { 285 this.setParam(opts, KEY_CONFIG_URLS, url.toString()); 286 } else { 287 this.setParam(opts, KEY_CONFIG_URLS, previousURLNames + "," + url); 288 } 289 } 290 } 291 292}