1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17 package org.apache.commons.vfs2.provider.hdfs;
18
19 import java.io.InputStream;
20 import java.net.MalformedURLException;
21 import java.net.URL;
22 import java.util.Arrays;
23 import java.util.stream.Stream;
24
25 import org.apache.commons.lang3.StringUtils;
26 import org.apache.commons.vfs2.FileSystem;
27 import org.apache.commons.vfs2.FileSystemConfigBuilder;
28 import org.apache.commons.vfs2.FileSystemOptions;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.Path;
31
32 /**
33 * Configuration settings for the HdfsFileSystem.
34 *
35 * @since 2.1
36 */
37 public final class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder {
38
39 private static final HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder();
40 private static final String KEY_CONFIG_NAMES = "configNames";
41 private static final String KEY_CONFIG_PATHS = "configPaths";
42 private static final String KEY_CONFIG_URLS = "configURLs";
43 private static final String KEY_CONFIG_STREAM = "configStream";
44 private static final String KEY_CONFIG_CONF = "configConf";
45
46 /**
47 * Gets the singleton instance.
48 *
49 * @return HdfsFileSystemConfigBuilder instance
50 */
51 public static HdfsFileSystemConfigBuilder getInstance() {
52 return BUILDER;
53 }
54
55 /**
56 * Constructs a new instance.
57 */
58 private HdfsFileSystemConfigBuilder() {
59 super("hdfs.");
60 }
61
62 /**
63 * @return HDFSFileSystem
64 */
65 @Override
66 protected Class<? extends FileSystem> getConfigClass() {
67 return HdfsFileSystem.class;
68 }
69
70 /**
71 * Gets the alternate configuration object.
72 *
73 * @return alternate configuration object or {@code null}.
74 * @param opts The FileSystemOptions.
75 * @see #setConfigConfiguration(FileSystemOptions, Configuration)
76 */
77 public Configuration getConfigConfiguration(final FileSystemOptions opts) {
78 return getParam(opts, KEY_CONFIG_CONF);
79 }
80
81 /**
82 * Gets the alternate configuration input stream.
83 *
84 * @return alternate configuration input stream or {@code null}.
85 * @param opts The FileSystemOptions.
86 * @see #setConfigInputStream(FileSystemOptions, InputStream)
87 */
88 public InputStream getConfigInputStream(final FileSystemOptions opts) {
89 return getParam(opts, KEY_CONFIG_STREAM);
90 }
91
92 /**
93 * Gets the names of alternate configuration resources.
94 *
95 * @return resource name list of alternate configurations or {@code null}.
96 * @param opts The FileSystemOptions.
97 * @see #setConfigName(FileSystemOptions, String)
98 */
99 public String[] getConfigNames(final FileSystemOptions opts) {
100 final String names = this.getString(opts, KEY_CONFIG_NAMES);
101 return StringUtils.isEmpty(names) ? null : names.split(",");
102 }
103
104 /**
105 * Gets the paths of alternate configuration file system files.
106 *
107 * @return list of full paths of alternate configuration files or {@code null}.
108 * @param opts The FileSystemOptions.
109 * @see #setConfigPath(FileSystemOptions, Path)
110 */
111 public Path[] getConfigPaths(final FileSystemOptions opts) {
112 final String pathNames = this.getString(opts, KEY_CONFIG_PATHS);
113 if (StringUtils.isEmpty(pathNames)) {
114 return null;
115 }
116 return Stream.of(pathNames.split(",")).map(Path::new).toArray(Path[]::new);
117 }
118
119 /**
120 * Gets URLs of alternate configurations.
121 *
122 * @return list of alternate configuration URLs or {@code null}.
123 * @param opts The FileSystemOptions.
124 * @see #setConfigURL(FileSystemOptions, URL)
125 */
126 public URL[] getConfigURLs(final FileSystemOptions opts) {
127 final String urlNames = this.getString(opts, KEY_CONFIG_URLS);
128 if (StringUtils.isEmpty(urlNames)) {
129 return null;
130 }
131 final String[] urls = urlNames.split(",");
132 final URL[] realURLs = new URL[urls.length];
133 Arrays.setAll(realURLs, i -> {
134 try {
135 return new URL(urls[i]);
136 } catch (final MalformedURLException e) {
137 // This should never happen because we save it in the proper form.
138 throw new IllegalArgumentException(urls[i], e);
139 }
140
141 });
142 return realURLs;
143 }
144
145 /**
146 * Sets the configuration object to be loaded after the defaults.
147 * <p>
148 * Specifies an already initialized configuration object to override any specific HDFS settings. The property will
149 * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Configuration)} after the URL was set as
150 * the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
151 * </p>
152 * <p>
153 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
154 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
155 * too.
156 * </p>
157 *
158 * @param opts The FileSystemOptions to modify.
159 * @param configuration additional configuration object or {@code null} to unset any configuration object previously
160 * set.
161 */
162 public void setConfigConfiguration(final FileSystemOptions opts, final Configuration configuration) {
163 this.setParam(opts, KEY_CONFIG_CONF, configuration);
164 }
165
166 /**
167 * Sets the input stream of configuration file to be loaded after the defaults.
168 * <p>
169 * Specifies an input stream connected to a config file to override any specific HDFS settings. The property will be
170 * passed on to {@code org.apache.hadoop.conf.Configuration#addResource(InputStream)} after the URL was set as the
171 * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
172 * </p>
173 * <p>
174 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
175 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
176 * too.
177 * </p>
178 *
179 * @param opts The FileSystemOptions to modify.
180 * @param inputStream input stream of additional configuration file or {@code null} to unset the configuration input
181 * stream previously set up.
182 */
183 public void setConfigInputStream(final FileSystemOptions opts, final InputStream inputStream) {
184 this.setParam(opts, KEY_CONFIG_STREAM, inputStream);
185 }
186
187 /**
188 * Sets the name of configuration resource to be loaded after the defaults.
189 * <p>
190 * Specifies the name of a config resource to override any specific HDFS settings. The property will be passed on to
191 * {@code org.apache.hadoop.conf.Configuration#addResource(String)} after the URL was set as the default name with:
192 * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
193 * </p>
194 * <p>
195 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
196 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
197 * too.
198 * </p>
199 * <p>
200 * This method may be called multiple times and all the specified resources will be loaded in the order they were
201 * specified.
202 * </p>
203 * <p>
204 * Note also, that if a list of names is provided, separated by commas ({@code ","}), that this will work the same
205 * as calling this method a number of times with just one name each.
206 * </p>
207 *
208 * @param opts The FileSystemOptions to modify.
209 * @param name resource name of additional configuration or {@code null} to unset all the values set so far.
210 * @see #getConfigNames
211 */
212 public void setConfigName(final FileSystemOptions opts, final String name) {
213 if (StringUtils.isEmpty(name)) {
214 this.setParam(opts, KEY_CONFIG_NAMES, null);
215 } else {
216 final String previousNames = this.getString(opts, KEY_CONFIG_NAMES);
217 if (previousNames == null || previousNames.isEmpty()) {
218 this.setParam(opts, KEY_CONFIG_NAMES, name);
219 } else {
220 this.setParam(opts, KEY_CONFIG_NAMES, previousNames + "," + name);
221 }
222 }
223 }
224
225 /**
226 * Sets the full path of configuration file to be loaded after the defaults.
227 * <p>
228 * Specifies the path of a local file system config file to override any specific HDFS settings. The property will
229 * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Path)} after the URL was set as the
230 * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
231 * </p>
232 * <p>
233 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
234 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
235 * too.
236 * </p>
237 * <p>
238 * This method may be called multiple times and all the specified resources will be loaded in the order they were
239 * specified.
240 * </p>
241 *
242 * @param opts The FileSystemOptions to modify.
243 * @param path full path of additional configuration file (local file system) or {@code null} to unset all the path
244 * values set so far.
245 */
246 public void setConfigPath(final FileSystemOptions opts, final Path path) {
247 if (path == null) {
248 this.setParam(opts, KEY_CONFIG_PATHS, null);
249 } else {
250 final String previousPathNames = this.getString(opts, KEY_CONFIG_PATHS);
251 if (StringUtils.isEmpty(previousPathNames)) {
252 this.setParam(opts, KEY_CONFIG_PATHS, path.toString());
253 } else {
254 this.setParam(opts, KEY_CONFIG_PATHS, previousPathNames + "," + path);
255 }
256 }
257 }
258
259 /**
260 * Sets the URL of configuration file to be loaded after the defaults.
261 * <p>
262 * Specifies the URL of a config file to override any specific HDFS settings. The property will be passed on to
263 * {@code org.apache.hadoop.conf.Configuration#addResource(URL)} after the URL was set as the default name with:
264 * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
265 * </p>
266 * <p>
267 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
268 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
269 * too.
270 * </p>
271 * <p>
272 * This method may be called multiple times and all the specified resources will be loaded in the order they were
273 * specified.
274 * </p>
275 *
276 * @param opts The FileSystemOptions to modify.
277 * @param url URL of additional configuration file or {@code null} to unset all the URL values set so far.
278 */
279 public void setConfigURL(final FileSystemOptions opts, final URL url) {
280 if (url == null) {
281 this.setParam(opts, KEY_CONFIG_URLS, null);
282 } else {
283 final String previousURLNames = this.getString(opts, KEY_CONFIG_URLS);
284 if (StringUtils.isEmpty(previousURLNames)) {
285 this.setParam(opts, KEY_CONFIG_URLS, url.toString());
286 } else {
287 this.setParam(opts, KEY_CONFIG_URLS, previousURLNames + "," + url);
288 }
289 }
290 }
291
292 }