View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.vfs2.provider.hdfs;
18  
19  import java.io.InputStream;
20  import java.net.MalformedURLException;
21  import java.net.URL;
22  import org.apache.commons.vfs2.FileSystem;
23  import org.apache.commons.vfs2.FileSystemConfigBuilder;
24  import org.apache.commons.vfs2.FileSystemOptions;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.Path;
27  
28  /**
29   * Configuration settings for the HdfsFileSystem.
30   *
31   * @since 2.1
32   */
33  public final class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder {
34  
35      private static final HdfsFileSystemConfigBuilderleSystemConfigBuilder.html#HdfsFileSystemConfigBuilder">HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder();
36      private static final String KEY_CONFIG_NAMES = "configNames";
37      private static final String KEY_CONFIG_PATHS = "configPaths";
38      private static final String KEY_CONFIG_URLS = "configURLs";
39      private static final String KEY_CONFIG_STREAM = "configStream";
40      private static final String KEY_CONFIG_CONF = "configConf";
41  
42      private HdfsFileSystemConfigBuilder() {
43          super("hdfs.");
44      }
45  
46      /**
47       * @return HdfsFileSystemConfigBuilder instance
48       */
49      public static HdfsFileSystemConfigBuilder getInstance() {
50          return BUILDER;
51      }
52  
53      /**
54       * @return HDFSFileSystem
55       */
56      @Override
57      protected Class<? extends FileSystem> getConfigClass() {
58          return HdfsFileSystem.class;
59      }
60  
61      /**
62       * Get names of alternate configuration resources.
63       *
64       * @return resource name list of alternate configurations or {@code null}.
65       * @param opts The FileSystemOptions.
66       * @see #setConfigName(FileSystemOptions, String)
67       */
68      public String[] getConfigNames(final FileSystemOptions opts) {
69          final String names = this.getString(opts, KEY_CONFIG_NAMES);
70          return names == null || names.isEmpty() ? null : names.split(",");
71      }
72  
73      /**
74       * Sets the name of configuration resource to be loaded after the defaults.
75       * <p>
76       * Specifies the name of a config resource to override any specific HDFS settings. The property will be passed on to
77       * {@code org.apache.hadoop.conf.Configuration#addResource(String)} after the URL was set as the default name with:
78       * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
79       * </p>
80       * <p>
81       * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
82       * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
83       * too.
84       * </p>
85       * <p>
86       * This method may be called multiple times and all the specified resources will be loaded in the order they were
87       * specified.
88       * </p>
89       * <p>
90       * Note also, that if a list of names is provided, separated by commas ({@code ","}), that this will work the same
91       * as calling this method a number of times with just one name each.
92       * </p>
93       *
94       * @param opts The FileSystemOptions to modify.
95       * @param name resource name of additional configuration or {@code null} to unset all the values set so far.
96       * @see #getConfigNames
97       */
98      public void setConfigName(final FileSystemOptions opts, final String name) {
99          if (name == null || name.isEmpty()) {
100             this.setParam(opts, KEY_CONFIG_NAMES, null);
101         } else {
102             final String previousNames = this.getString(opts, KEY_CONFIG_NAMES);
103             if (previousNames == null || previousNames.isEmpty()) {
104                 this.setParam(opts, KEY_CONFIG_NAMES, name);
105             } else {
106                 this.setParam(opts, KEY_CONFIG_NAMES, previousNames + "," + name);
107             }
108         }
109     }
110 
111     /**
112      * Get paths of alternate configuration file system files.
113      *
114      * @return list of full paths of alternate configuration files or {@code null}.
115      * @param opts The FileSystemOptions.
116      * @see #setConfigPath(FileSystemOptions, Path)
117      */
118     public Path[] getConfigPaths(final FileSystemOptions opts) {
119         final String pathNames = this.getString(opts, KEY_CONFIG_PATHS);
120         if (pathNames == null || pathNames.isEmpty()) {
121             return null;
122         }
123         final String[] paths = pathNames.split(",");
124         final Path[] realPaths = new Path[paths.length];
125         for (int i = 0; i < paths.length; i++) {
126             realPaths[i] = new Path(paths[i]);
127         }
128         return realPaths;
129     }
130 
131     /**
132      * Sets the full path of configuration file to be loaded after the defaults.
133      * <p>
134      * Specifies the path of a local file system config file to override any specific HDFS settings. The property will
135      * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Path)} after the URL was set as the
136      * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
137      * <p>
138      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
139      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
140      * too.
141      * <p>
142      * This method may be called multiple times and all the specified resources will be loaded in the order they were
143      * specified.
144      *
145      * @param opts The FileSystemOptions to modify.
146      * @param path full path of additional configuration file (local file system) or {@code null} to unset all the path
147      *            values set so far.
148      */
149     public void setConfigPath(final FileSystemOptions opts, final Path path) {
150         if (path == null) {
151             this.setParam(opts, KEY_CONFIG_PATHS, null);
152         } else {
153             final String previousPathNames = this.getString(opts, KEY_CONFIG_PATHS);
154             if (previousPathNames == null || previousPathNames.isEmpty()) {
155                 this.setParam(opts, KEY_CONFIG_PATHS, path.toString());
156             } else {
157                 this.setParam(opts, KEY_CONFIG_PATHS, previousPathNames + "," + path.toString());
158             }
159         }
160     }
161 
162     /**
163      * Get URLs of alternate configurations.
164      *
165      * @return list of alternate configuration URLs or {@code null}.
166      * @param opts The FileSystemOptions.
167      * @see #setConfigURL(FileSystemOptions, URL)
168      */
169     public URL[] getConfigURLs(final FileSystemOptions opts) {
170         try {
171             final String urlNames = this.getString(opts, KEY_CONFIG_URLS);
172             if (urlNames == null || urlNames.isEmpty()) {
173                 return null;
174             }
175             final String[] urls = urlNames.split(",");
176             final URL[] realURLs = new URL[urls.length];
177             for (int i = 0; i < urls.length; i++) {
178                 realURLs[i] = new URL(urls[i]);
179             }
180             return realURLs;
181         } catch (final MalformedURLException mue) {
182             // This should never happen because we save it in the proper form
183         }
184         return null;
185     }
186 
187     /**
188      * Sets the URL of configuration file to be loaded after the defaults.
189      * <p>
190      * Specifies the URL of a config file to override any specific HDFS settings. The property will be passed on to
191      * {@code org.apache.hadoop.conf.Configuration#addResource(URL)} after the URL was set as the default name with:
192      * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
193      * <p>
194      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
195      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
196      * too.
197      * <p>
198      * This method may be called multiple times and all the specified resources will be loaded in the order they were
199      * specified.
200      *
201      * @param opts The FileSystemOptions to modify.
202      * @param url URL of additional configuration file or {@code null} to unset all the URL values set so far.
203      */
204     public void setConfigURL(final FileSystemOptions opts, final URL url) {
205         if (url == null) {
206             this.setParam(opts, KEY_CONFIG_URLS, null);
207         } else {
208             final String previousURLNames = this.getString(opts, KEY_CONFIG_URLS);
209             if (previousURLNames == null || previousURLNames.isEmpty()) {
210                 this.setParam(opts, KEY_CONFIG_URLS, url.toString());
211             } else {
212                 this.setParam(opts, KEY_CONFIG_URLS, previousURLNames + "," + url.toString());
213             }
214         }
215     }
216 
217     /**
218      * Get alternate configuration input stream.
219      *
220      * @return alternate configuration input stream or {@code null}.
221      * @param opts The FileSystemOptions.
222      * @see #setConfigInputStream(FileSystemOptions, InputStream)
223      */
224     public InputStream getConfigInputStream(final FileSystemOptions opts) {
225         return (InputStream) this.getParam(opts, KEY_CONFIG_STREAM);
226     }
227 
228     /**
229      * Sets the input stream of configuration file to be loaded after the defaults.
230      * <p>
231      * Specifies an input stream connected to a config file to override any specific HDFS settings. The property will be
232      * passed on to {@code org.apache.hadoop.conf.Configuration#addResource(InputStream)} after the URL was set as the
233      * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
234      * <p>
235      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
236      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
237      * too.
238      *
239      * @param opts The FileSystemOptions to modify.
240      * @param inputStream input stream of additional configuration file or {@code null} to unset the configuration input
241      *            stream previously set up.
242      */
243     public void setConfigInputStream(final FileSystemOptions opts, final InputStream inputStream) {
244         this.setParam(opts, KEY_CONFIG_STREAM, inputStream);
245     }
246 
247     /**
248      * Get alternate configuration object.
249      *
250      * @return alternate configuration object or {@code null}.
251      * @param opts The FileSystemOptions.
252      * @see #setConfigConfiguration(FileSystemOptions, Configuration)
253      */
254     public Configuration getConfigConfiguration(final FileSystemOptions opts) {
255         return (Configuration) this.getParam(opts, KEY_CONFIG_CONF);
256     }
257 
258     /**
259      * Sets the configuration object to be loaded after the defaults.
260      * <p>
261      * Specifies an already initialized configuration object to override any specific HDFS settings. The property will
262      * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Configuration)} after the URL was set as
263      * the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
264      * <p>
265      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
266      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
267      * too.
268      *
269      * @param opts The FileSystemOptions to modify.
270      * @param configuration additional configuration object or {@code null} to unset any configuration object previously
271      *            set.
272      */
273     public void setConfigConfiguration(final FileSystemOptions opts, final Configuration configuration) {
274         this.setParam(opts, KEY_CONFIG_CONF, configuration);
275     }
276 
277 }