View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.vfs2.provider.hdfs;
18  
19  import java.io.InputStream;
20  import java.net.MalformedURLException;
21  import java.net.URL;
22  import java.util.stream.Stream;
23  
24  import org.apache.commons.lang3.StringUtils;
25  import org.apache.commons.vfs2.FileSystem;
26  import org.apache.commons.vfs2.FileSystemConfigBuilder;
27  import org.apache.commons.vfs2.FileSystemOptions;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.Path;
30  
31  /**
32   * Configuration settings for the HdfsFileSystem.
33   *
34   * @since 2.1
35   */
36  public final class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder {
37  
38      private static final HdfsFileSystemConfigBuilderleSystemConfigBuilder.html#HdfsFileSystemConfigBuilder">HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder();
39      private static final String KEY_CONFIG_NAMES = "configNames";
40      private static final String KEY_CONFIG_PATHS = "configPaths";
41      private static final String KEY_CONFIG_URLS = "configURLs";
42      private static final String KEY_CONFIG_STREAM = "configStream";
43      private static final String KEY_CONFIG_CONF = "configConf";
44  
45      /**
46       * @return HdfsFileSystemConfigBuilder instance
47       */
48      public static HdfsFileSystemConfigBuilder getInstance() {
49          return BUILDER;
50      }
51  
52      private HdfsFileSystemConfigBuilder() {
53          super("hdfs.");
54      }
55  
56      /**
57       * @return HDFSFileSystem
58       */
59      @Override
60      protected Class<? extends FileSystem> getConfigClass() {
61          return HdfsFileSystem.class;
62      }
63  
64      /**
65       * Get alternate configuration object.
66       *
67       * @return alternate configuration object or {@code null}.
68       * @param opts The FileSystemOptions.
69       * @see #setConfigConfiguration(FileSystemOptions, Configuration)
70       */
71      public Configuration getConfigConfiguration(final FileSystemOptions opts) {
72          return getParam(opts, KEY_CONFIG_CONF);
73      }
74  
75      /**
76       * Get alternate configuration input stream.
77       *
78       * @return alternate configuration input stream or {@code null}.
79       * @param opts The FileSystemOptions.
80       * @see #setConfigInputStream(FileSystemOptions, InputStream)
81       */
82      public InputStream getConfigInputStream(final FileSystemOptions opts) {
83          return getParam(opts, KEY_CONFIG_STREAM);
84      }
85  
86      /**
87       * Get names of alternate configuration resources.
88       *
89       * @return resource name list of alternate configurations or {@code null}.
90       * @param opts The FileSystemOptions.
91       * @see #setConfigName(FileSystemOptions, String)
92       */
93      public String[] getConfigNames(final FileSystemOptions opts) {
94          final String names = this.getString(opts, KEY_CONFIG_NAMES);
95          return StringUtils.isEmpty(names) ? null : names.split(",");
96      }
97  
98      /**
99       * Get paths of alternate configuration file system files.
100      *
101      * @return list of full paths of alternate configuration files or {@code null}.
102      * @param opts The FileSystemOptions.
103      * @see #setConfigPath(FileSystemOptions, Path)
104      */
105     public Path[] getConfigPaths(final FileSystemOptions opts) {
106         final String pathNames = this.getString(opts, KEY_CONFIG_PATHS);
107         if (StringUtils.isEmpty(pathNames)) {
108             return null;
109         }
110         return Stream.of(pathNames.split(",")).map(Path::new).toArray(Path[]::new);
111     }
112 
113     /**
114      * Get URLs of alternate configurations.
115      *
116      * @return list of alternate configuration URLs or {@code null}.
117      * @param opts The FileSystemOptions.
118      * @see #setConfigURL(FileSystemOptions, URL)
119      */
120     public URL[] getConfigURLs(final FileSystemOptions opts) {
121         try {
122             final String urlNames = this.getString(opts, KEY_CONFIG_URLS);
123             if (StringUtils.isEmpty(urlNames)) {
124                 return null;
125             }
126             final String[] urls = urlNames.split(",");
127             final URL[] realURLs = new URL[urls.length];
128             for (int i = 0; i < urls.length; i++) {
129                 realURLs[i] = new URL(urls[i]);
130             }
131             return realURLs;
132         } catch (final MalformedURLException mue) {
133             // This should never happen because we save it in the proper form
134         }
135         return null;
136     }
137 
138     /**
139      * Sets the configuration object to be loaded after the defaults.
140      * <p>
141      * Specifies an already initialized configuration object to override any specific HDFS settings. The property will
142      * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Configuration)} after the URL was set as
143      * the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
144      * <p>
145      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
146      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
147      * too.
148      *
149      * @param opts The FileSystemOptions to modify.
150      * @param configuration additional configuration object or {@code null} to unset any configuration object previously
151      *            set.
152      */
153     public void setConfigConfiguration(final FileSystemOptions opts, final Configuration configuration) {
154         this.setParam(opts, KEY_CONFIG_CONF, configuration);
155     }
156 
157     /**
158      * Sets the input stream of configuration file to be loaded after the defaults.
159      * <p>
160      * Specifies an input stream connected to a config file to override any specific HDFS settings. The property will be
161      * passed on to {@code org.apache.hadoop.conf.Configuration#addResource(InputStream)} after the URL was set as the
162      * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
163      * <p>
164      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
165      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
166      * too.
167      *
168      * @param opts The FileSystemOptions to modify.
169      * @param inputStream input stream of additional configuration file or {@code null} to unset the configuration input
170      *            stream previously set up.
171      */
172     public void setConfigInputStream(final FileSystemOptions opts, final InputStream inputStream) {
173         this.setParam(opts, KEY_CONFIG_STREAM, inputStream);
174     }
175 
176     /**
177      * Sets the name of configuration resource to be loaded after the defaults.
178      * <p>
179      * Specifies the name of a config resource to override any specific HDFS settings. The property will be passed on to
180      * {@code org.apache.hadoop.conf.Configuration#addResource(String)} after the URL was set as the default name with:
181      * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
182      * </p>
183      * <p>
184      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
185      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
186      * too.
187      * </p>
188      * <p>
189      * This method may be called multiple times and all the specified resources will be loaded in the order they were
190      * specified.
191      * </p>
192      * <p>
193      * Note also, that if a list of names is provided, separated by commas ({@code ","}), that this will work the same
194      * as calling this method a number of times with just one name each.
195      * </p>
196      *
197      * @param opts The FileSystemOptions to modify.
198      * @param name resource name of additional configuration or {@code null} to unset all the values set so far.
199      * @see #getConfigNames
200      */
201     public void setConfigName(final FileSystemOptions opts, final String name) {
202         if (StringUtils.isEmpty(name)) {
203             this.setParam(opts, KEY_CONFIG_NAMES, null);
204         } else {
205             final String previousNames = this.getString(opts, KEY_CONFIG_NAMES);
206             if (previousNames == null || previousNames.isEmpty()) {
207                 this.setParam(opts, KEY_CONFIG_NAMES, name);
208             } else {
209                 this.setParam(opts, KEY_CONFIG_NAMES, previousNames + "," + name);
210             }
211         }
212     }
213 
214     /**
215      * Sets the full path of configuration file to be loaded after the defaults.
216      * <p>
217      * Specifies the path of a local file system config file to override any specific HDFS settings. The property will
218      * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Path)} after the URL was set as the
219      * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
220      * <p>
221      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
222      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
223      * too.
224      * <p>
225      * This method may be called multiple times and all the specified resources will be loaded in the order they were
226      * specified.
227      *
228      * @param opts The FileSystemOptions to modify.
229      * @param path full path of additional configuration file (local file system) or {@code null} to unset all the path
230      *            values set so far.
231      */
232     public void setConfigPath(final FileSystemOptions opts, final Path path) {
233         if (path == null) {
234             this.setParam(opts, KEY_CONFIG_PATHS, null);
235         } else {
236             final String previousPathNames = this.getString(opts, KEY_CONFIG_PATHS);
237             if (StringUtils.isEmpty(previousPathNames)) {
238                 this.setParam(opts, KEY_CONFIG_PATHS, path.toString());
239             } else {
240                 this.setParam(opts, KEY_CONFIG_PATHS, previousPathNames + "," + path.toString());
241             }
242         }
243     }
244 
245     /**
246      * Sets the URL of configuration file to be loaded after the defaults.
247      * <p>
248      * Specifies the URL of a config file to override any specific HDFS settings. The property will be passed on to
249      * {@code org.apache.hadoop.conf.Configuration#addResource(URL)} after the URL was set as the default name with:
250      * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
251      * <p>
252      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
253      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
254      * too.
255      * <p>
256      * This method may be called multiple times and all the specified resources will be loaded in the order they were
257      * specified.
258      *
259      * @param opts The FileSystemOptions to modify.
260      * @param url URL of additional configuration file or {@code null} to unset all the URL values set so far.
261      */
262     public void setConfigURL(final FileSystemOptions opts, final URL url) {
263         if (url == null) {
264             this.setParam(opts, KEY_CONFIG_URLS, null);
265         } else {
266             final String previousURLNames = this.getString(opts, KEY_CONFIG_URLS);
267             if (StringUtils.isEmpty(previousURLNames)) {
268                 this.setParam(opts, KEY_CONFIG_URLS, url.toString());
269             } else {
270                 this.setParam(opts, KEY_CONFIG_URLS, previousURLNames + "," + url.toString());
271             }
272         }
273     }
274 
275 }