1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17 package org.apache.commons.vfs2.provider.hdfs;
18
19 import java.io.InputStream;
20 import java.net.MalformedURLException;
21 import java.net.URL;
22 import java.util.stream.Stream;
23
24 import org.apache.commons.lang3.StringUtils;
25 import org.apache.commons.vfs2.FileSystem;
26 import org.apache.commons.vfs2.FileSystemConfigBuilder;
27 import org.apache.commons.vfs2.FileSystemOptions;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.fs.Path;
30
31 /**
32 * Configuration settings for the HdfsFileSystem.
33 *
34 * @since 2.1
35 */
36 public final class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder {
37
38 private static final HdfsFileSystemConfigBuilderleSystemConfigBuilder.html#HdfsFileSystemConfigBuilder">HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder();
39 private static final String KEY_CONFIG_NAMES = "configNames";
40 private static final String KEY_CONFIG_PATHS = "configPaths";
41 private static final String KEY_CONFIG_URLS = "configURLs";
42 private static final String KEY_CONFIG_STREAM = "configStream";
43 private static final String KEY_CONFIG_CONF = "configConf";
44
45 /**
46 * @return HdfsFileSystemConfigBuilder instance
47 */
48 public static HdfsFileSystemConfigBuilder getInstance() {
49 return BUILDER;
50 }
51
52 private HdfsFileSystemConfigBuilder() {
53 super("hdfs.");
54 }
55
56 /**
57 * @return HDFSFileSystem
58 */
59 @Override
60 protected Class<? extends FileSystem> getConfigClass() {
61 return HdfsFileSystem.class;
62 }
63
64 /**
65 * Get alternate configuration object.
66 *
67 * @return alternate configuration object or {@code null}.
68 * @param opts The FileSystemOptions.
69 * @see #setConfigConfiguration(FileSystemOptions, Configuration)
70 */
71 public Configuration getConfigConfiguration(final FileSystemOptions opts) {
72 return getParam(opts, KEY_CONFIG_CONF);
73 }
74
75 /**
76 * Get alternate configuration input stream.
77 *
78 * @return alternate configuration input stream or {@code null}.
79 * @param opts The FileSystemOptions.
80 * @see #setConfigInputStream(FileSystemOptions, InputStream)
81 */
82 public InputStream getConfigInputStream(final FileSystemOptions opts) {
83 return getParam(opts, KEY_CONFIG_STREAM);
84 }
85
86 /**
87 * Get names of alternate configuration resources.
88 *
89 * @return resource name list of alternate configurations or {@code null}.
90 * @param opts The FileSystemOptions.
91 * @see #setConfigName(FileSystemOptions, String)
92 */
93 public String[] getConfigNames(final FileSystemOptions opts) {
94 final String names = this.getString(opts, KEY_CONFIG_NAMES);
95 return StringUtils.isEmpty(names) ? null : names.split(",");
96 }
97
98 /**
99 * Get paths of alternate configuration file system files.
100 *
101 * @return list of full paths of alternate configuration files or {@code null}.
102 * @param opts The FileSystemOptions.
103 * @see #setConfigPath(FileSystemOptions, Path)
104 */
105 public Path[] getConfigPaths(final FileSystemOptions opts) {
106 final String pathNames = this.getString(opts, KEY_CONFIG_PATHS);
107 if (StringUtils.isEmpty(pathNames)) {
108 return null;
109 }
110 return Stream.of(pathNames.split(",")).map(Path::new).toArray(Path[]::new);
111 }
112
113 /**
114 * Get URLs of alternate configurations.
115 *
116 * @return list of alternate configuration URLs or {@code null}.
117 * @param opts The FileSystemOptions.
118 * @see #setConfigURL(FileSystemOptions, URL)
119 */
120 public URL[] getConfigURLs(final FileSystemOptions opts) {
121 try {
122 final String urlNames = this.getString(opts, KEY_CONFIG_URLS);
123 if (StringUtils.isEmpty(urlNames)) {
124 return null;
125 }
126 final String[] urls = urlNames.split(",");
127 final URL[] realURLs = new URL[urls.length];
128 for (int i = 0; i < urls.length; i++) {
129 realURLs[i] = new URL(urls[i]);
130 }
131 return realURLs;
132 } catch (final MalformedURLException mue) {
133 // This should never happen because we save it in the proper form
134 }
135 return null;
136 }
137
138 /**
139 * Sets the configuration object to be loaded after the defaults.
140 * <p>
141 * Specifies an already initialized configuration object to override any specific HDFS settings. The property will
142 * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Configuration)} after the URL was set as
143 * the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
144 * <p>
145 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
146 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
147 * too.
148 *
149 * @param opts The FileSystemOptions to modify.
150 * @param configuration additional configuration object or {@code null} to unset any configuration object previously
151 * set.
152 */
153 public void setConfigConfiguration(final FileSystemOptions opts, final Configuration configuration) {
154 this.setParam(opts, KEY_CONFIG_CONF, configuration);
155 }
156
157 /**
158 * Sets the input stream of configuration file to be loaded after the defaults.
159 * <p>
160 * Specifies an input stream connected to a config file to override any specific HDFS settings. The property will be
161 * passed on to {@code org.apache.hadoop.conf.Configuration#addResource(InputStream)} after the URL was set as the
162 * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
163 * <p>
164 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
165 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
166 * too.
167 *
168 * @param opts The FileSystemOptions to modify.
169 * @param inputStream input stream of additional configuration file or {@code null} to unset the configuration input
170 * stream previously set up.
171 */
172 public void setConfigInputStream(final FileSystemOptions opts, final InputStream inputStream) {
173 this.setParam(opts, KEY_CONFIG_STREAM, inputStream);
174 }
175
176 /**
177 * Sets the name of configuration resource to be loaded after the defaults.
178 * <p>
179 * Specifies the name of a config resource to override any specific HDFS settings. The property will be passed on to
180 * {@code org.apache.hadoop.conf.Configuration#addResource(String)} after the URL was set as the default name with:
181 * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
182 * </p>
183 * <p>
184 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
185 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
186 * too.
187 * </p>
188 * <p>
189 * This method may be called multiple times and all the specified resources will be loaded in the order they were
190 * specified.
191 * </p>
192 * <p>
193 * Note also, that if a list of names is provided, separated by commas ({@code ","}), that this will work the same
194 * as calling this method a number of times with just one name each.
195 * </p>
196 *
197 * @param opts The FileSystemOptions to modify.
198 * @param name resource name of additional configuration or {@code null} to unset all the values set so far.
199 * @see #getConfigNames
200 */
201 public void setConfigName(final FileSystemOptions opts, final String name) {
202 if (StringUtils.isEmpty(name)) {
203 this.setParam(opts, KEY_CONFIG_NAMES, null);
204 } else {
205 final String previousNames = this.getString(opts, KEY_CONFIG_NAMES);
206 if (previousNames == null || previousNames.isEmpty()) {
207 this.setParam(opts, KEY_CONFIG_NAMES, name);
208 } else {
209 this.setParam(opts, KEY_CONFIG_NAMES, previousNames + "," + name);
210 }
211 }
212 }
213
214 /**
215 * Sets the full path of configuration file to be loaded after the defaults.
216 * <p>
217 * Specifies the path of a local file system config file to override any specific HDFS settings. The property will
218 * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Path)} after the URL was set as the
219 * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
220 * <p>
221 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
222 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
223 * too.
224 * <p>
225 * This method may be called multiple times and all the specified resources will be loaded in the order they were
226 * specified.
227 *
228 * @param opts The FileSystemOptions to modify.
229 * @param path full path of additional configuration file (local file system) or {@code null} to unset all the path
230 * values set so far.
231 */
232 public void setConfigPath(final FileSystemOptions opts, final Path path) {
233 if (path == null) {
234 this.setParam(opts, KEY_CONFIG_PATHS, null);
235 } else {
236 final String previousPathNames = this.getString(opts, KEY_CONFIG_PATHS);
237 if (StringUtils.isEmpty(previousPathNames)) {
238 this.setParam(opts, KEY_CONFIG_PATHS, path.toString());
239 } else {
240 this.setParam(opts, KEY_CONFIG_PATHS, previousPathNames + "," + path.toString());
241 }
242 }
243 }
244
245 /**
246 * Sets the URL of configuration file to be loaded after the defaults.
247 * <p>
248 * Specifies the URL of a config file to override any specific HDFS settings. The property will be passed on to
249 * {@code org.apache.hadoop.conf.Configuration#addResource(URL)} after the URL was set as the default name with:
250 * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
251 * <p>
252 * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
253 * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
254 * too.
255 * <p>
256 * This method may be called multiple times and all the specified resources will be loaded in the order they were
257 * specified.
258 *
259 * @param opts The FileSystemOptions to modify.
260 * @param url URL of additional configuration file or {@code null} to unset all the URL values set so far.
261 */
262 public void setConfigURL(final FileSystemOptions opts, final URL url) {
263 if (url == null) {
264 this.setParam(opts, KEY_CONFIG_URLS, null);
265 } else {
266 final String previousURLNames = this.getString(opts, KEY_CONFIG_URLS);
267 if (StringUtils.isEmpty(previousURLNames)) {
268 this.setParam(opts, KEY_CONFIG_URLS, url.toString());
269 } else {
270 this.setParam(opts, KEY_CONFIG_URLS, previousURLNames + "," + url.toString());
271 }
272 }
273 }
274
275 }