View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.vfs2.provider.hdfs;
18  
19  import java.io.InputStream;
20  import java.net.MalformedURLException;
21  import java.net.URL;
22  
23  import org.apache.commons.vfs2.FileSystem;
24  import org.apache.commons.vfs2.FileSystemConfigBuilder;
25  import org.apache.commons.vfs2.FileSystemOptions;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.Path;
28  
29  /**
30   * Configuration settings for the HdfsFileSystem.
31   *
32   * @since 2.1
33   */
34  public final class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder {
35  
36      private static final HdfsFileSystemConfigBuilderleSystemConfigBuilder.html#HdfsFileSystemConfigBuilder">HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder();
37      private static final String KEY_CONFIG_NAMES = "configNames";
38      private static final String KEY_CONFIG_PATHS = "configPaths";
39      private static final String KEY_CONFIG_URLS = "configURLs";
40      private static final String KEY_CONFIG_STREAM = "configStream";
41      private static final String KEY_CONFIG_CONF = "configConf";
42  
43      private HdfsFileSystemConfigBuilder() {
44          super("hdfs.");
45      }
46  
47      /**
48       * @return HdfsFileSystemConfigBuilder instance
49       */
50      public static HdfsFileSystemConfigBuilder getInstance() {
51          return BUILDER;
52      }
53  
54      /**
55       * @return HDFSFileSystem
56       */
57      @Override
58      protected Class<? extends FileSystem> getConfigClass() {
59          return HdfsFileSystem.class;
60      }
61  
62      /**
63       * Get names of alternate configuration resources.
64       *
65       * @return resource name list of alternate configurations or {@code null}.
66       * @param opts The FileSystemOptions.
67       * @see #setConfigName(FileSystemOptions, String)
68       */
69      public String[] getConfigNames(final FileSystemOptions opts) {
70          final String names = this.getString(opts, KEY_CONFIG_NAMES);
71          return names == null || names.isEmpty() ? null : names.split(",");
72      }
73  
74      /**
75       * Sets the name of configuration resource to be loaded after the defaults.
76       * <p>
77       * Specifies the name of a config resource to override any specific HDFS settings. The property will be passed on to
78       * {@code org.apache.hadoop.conf.Configuration#addResource(String)} after the URL was set as the default name with:
79       * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
80       * </p>
81       * <p>
82       * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
83       * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
84       * too.
85       * </p>
86       * <p>
87       * This method may be called multiple times and all the specified resources will be loaded in the order they were
88       * specified.
89       * </p>
90       * <p>
91       * Note also, that if a list of names is provided, separated by commas ({@code ","}), that this will work the same
92       * as calling this method a number of times with just one name each.
93       * </p>
94       *
95       * @param opts The FileSystemOptions to modify.
96       * @param name resource name of additional configuration or {@code null} to unset all the values set so far.
97       * @see #getConfigNames
98       */
99      public void setConfigName(final FileSystemOptions opts, final String name) {
100         if (name == null || name.isEmpty()) {
101             this.setParam(opts, KEY_CONFIG_NAMES, null);
102         } else {
103             final String previousNames = this.getString(opts, KEY_CONFIG_NAMES);
104             if (previousNames == null || previousNames.isEmpty()) {
105                 this.setParam(opts, KEY_CONFIG_NAMES, name);
106             } else {
107                 this.setParam(opts, KEY_CONFIG_NAMES, previousNames + "," + name);
108             }
109         }
110     }
111 
112     /**
113      * Get paths of alternate configuration file system files.
114      *
115      * @return list of full paths of alternate configuration files or {@code null}.
116      * @param opts The FileSystemOptions.
117      * @see #setConfigPath(FileSystemOptions, Path)
118      */
119     public Path[] getConfigPaths(final FileSystemOptions opts) {
120         final String pathNames = this.getString(opts, KEY_CONFIG_PATHS);
121         if (pathNames == null || pathNames.isEmpty()) {
122             return null;
123         }
124         final String[] paths = pathNames.split(",");
125         final Path[] realPaths = new Path[paths.length];
126         for (int i = 0; i < paths.length; i++) {
127             realPaths[i] = new Path(paths[i]);
128         }
129         return realPaths;
130     }
131 
132     /**
133      * Sets the full path of configuration file to be loaded after the defaults.
134      * <p>
135      * Specifies the path of a local file system config file to override any specific HDFS settings. The property will
136      * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Path)} after the URL was set as the
137      * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
138      * <p>
139      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
140      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
141      * too.
142      * <p>
143      * This method may be called multiple times and all the specified resources will be loaded in the order they were
144      * specified.
145      *
146      * @param opts The FileSystemOptions to modify.
147      * @param path full path of additional configuration file (local file system) or {@code null} to unset all the path
148      *            values set so far.
149      */
150     public void setConfigPath(final FileSystemOptions opts, final Path path) {
151         if (path == null) {
152             this.setParam(opts, KEY_CONFIG_PATHS, null);
153         } else {
154             final String previousPathNames = this.getString(opts, KEY_CONFIG_PATHS);
155             if (previousPathNames == null || previousPathNames.isEmpty()) {
156                 this.setParam(opts, KEY_CONFIG_PATHS, path.toString());
157             } else {
158                 this.setParam(opts, KEY_CONFIG_PATHS, previousPathNames + "," + path.toString());
159             }
160         }
161     }
162 
163     /**
164      * Get URLs of alternate configurations.
165      *
166      * @return list of alternate configuration URLs or {@code null}.
167      * @param opts The FileSystemOptions.
168      * @see #setConfigURL(FileSystemOptions, URL)
169      */
170     public URL[] getConfigURLs(final FileSystemOptions opts) {
171         try {
172             final String urlNames = this.getString(opts, KEY_CONFIG_URLS);
173             if (urlNames == null || urlNames.isEmpty()) {
174                 return null;
175             }
176             final String[] urls = urlNames.split(",");
177             final URL[] realURLs = new URL[urls.length];
178             for (int i = 0; i < urls.length; i++) {
179                 realURLs[i] = new URL(urls[i]);
180             }
181             return realURLs;
182         } catch (final MalformedURLException mue) {
183             // This should never happen because we save it in the proper form
184         }
185         return null;
186     }
187 
188     /**
189      * Sets the URL of configuration file to be loaded after the defaults.
190      * <p>
191      * Specifies the URL of a config file to override any specific HDFS settings. The property will be passed on to
192      * {@code org.apache.hadoop.conf.Configuration#addResource(URL)} after the URL was set as the default name with:
193      * {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
194      * <p>
195      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
196      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
197      * too.
198      * <p>
199      * This method may be called multiple times and all the specified resources will be loaded in the order they were
200      * specified.
201      *
202      * @param opts The FileSystemOptions to modify.
203      * @param url URL of additional configuration file or {@code null} to unset all the URL values set so far.
204      */
205     public void setConfigURL(final FileSystemOptions opts, final URL url) {
206         if (url == null) {
207             this.setParam(opts, KEY_CONFIG_URLS, null);
208         } else {
209             final String previousURLNames = this.getString(opts, KEY_CONFIG_URLS);
210             if (previousURLNames == null || previousURLNames.isEmpty()) {
211                 this.setParam(opts, KEY_CONFIG_URLS, url.toString());
212             } else {
213                 this.setParam(opts, KEY_CONFIG_URLS, previousURLNames + "," + url.toString());
214             }
215         }
216     }
217 
218     /**
219      * Get alternate configuration input stream.
220      *
221      * @return alternate configuration input stream or {@code null}.
222      * @param opts The FileSystemOptions.
223      * @see #setConfigInputStream(FileSystemOptions, InputStream)
224      */
225     public InputStream getConfigInputStream(final FileSystemOptions opts) {
226         return (InputStream) this.getParam(opts, KEY_CONFIG_STREAM);
227     }
228 
229     /**
230      * Sets the input stream of configuration file to be loaded after the defaults.
231      * <p>
232      * Specifies an input stream connected to a config file to override any specific HDFS settings. The property will be
233      * passed on to {@code org.apache.hadoop.conf.Configuration#addResource(InputStream)} after the URL was set as the
234      * default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
235      * <p>
236      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
237      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
238      * too.
239      *
240      * @param opts The FileSystemOptions to modify.
241      * @param inputStream input stream of additional configuration file or {@code null} to unset the configuration input
242      *            stream previously set up.
243      */
244     public void setConfigInputStream(final FileSystemOptions opts, final InputStream inputStream) {
245         this.setParam(opts, KEY_CONFIG_STREAM, inputStream);
246     }
247 
248     /**
249      * Get alternate configuration object.
250      *
251      * @return alternate configuration object or {@code null}.
252      * @param opts The FileSystemOptions.
253      * @see #setConfigConfiguration(FileSystemOptions, Configuration)
254      */
255     public Configuration getConfigConfiguration(final FileSystemOptions opts) {
256         return (Configuration) this.getParam(opts, KEY_CONFIG_CONF);
257     }
258 
259     /**
260      * Sets the configuration object to be loaded after the defaults.
261      * <p>
262      * Specifies an already initialized configuration object to override any specific HDFS settings. The property will
263      * be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Configuration)} after the URL was set as
264      * the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
265      * <p>
266      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname} property in order
267      * to access HDFS files stored in an AWS installation (from outside their firewall). There are other possible uses
268      * too.
269      *
270      * @param opts The FileSystemOptions to modify.
271      * @param configuration additional configuration object or {@code null} to unset any configuration object previously
272      *            set.
273      */
274     public void setConfigConfiguration(final FileSystemOptions opts, final Configuration configuration) {
275         this.setParam(opts, KEY_CONFIG_CONF, configuration);
276     }
277 
278 }