View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.vfs2.provider.hdfs;
18  
19  import java.io.InputStream;
20  import java.net.MalformedURLException;
21  import java.net.URL;
22  import org.apache.commons.vfs2.FileSystem;
23  import org.apache.commons.vfs2.FileSystemConfigBuilder;
24  import org.apache.commons.vfs2.FileSystemOptions;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.Path;
27  
28  /**
29   * Configuration settings for the HdfsFileSystem.
30   *
31   * @since 2.1
32   */
33  public final class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder
34  {
35      private static final HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder();
36      private static final String KEY_CONFIG_NAMES  = "configNames";
37      private static final String KEY_CONFIG_PATHS  = "configPaths";
38      private static final String KEY_CONFIG_URLS   = "configURLs";
39      private static final String KEY_CONFIG_STREAM = "configStream";
40      private static final String KEY_CONFIG_CONF   = "configConf";
41  
42      private HdfsFileSystemConfigBuilder()
43      {
44          super("hdfs.");
45      }
46  
47      /**
48       * @return HdfsFileSystemConfigBuilder instance
49       */
50      public static HdfsFileSystemConfigBuilder getInstance()
51      {
52          return BUILDER;
53      }
54  
55      /**
56       * @return HDFSFileSystem
57       */
58      @Override
59      protected Class<? extends FileSystem> getConfigClass()
60      {
61          return HdfsFileSystem.class;
62      }
63  
64      /**
65       * Get names of alternate configuration resources.
66       *
67       * @return resource name list of alternate configurations or {@code null}.
68       * @param opts The FileSystemOptions.
69       * @see #setConfigName(FileSystemOptions, String)
70       */
71      public String[] getConfigNames(final FileSystemOptions opts)
72      {
73          String names = this.getString(opts, KEY_CONFIG_NAMES);
74          return names == null || names.isEmpty() ? null : names.split(",");
75      }
76  
77      /**
78       * Sets the name of configuration resource to be loaded after the defaults.
79       * <p>
80       * Specifies the name of a config resource to override any specific HDFS settings.
81       * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(String)}
82       * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
83       * <p>
84       * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
85       * property in order to access HDFS files stored in an AWS installation (from outside their
86       * firewall). There are other possible uses too.
87       * <p>
88       * This method may be called multiple times and all the specified resources will be loaded
89       * in the order they were specified.
90       * <p>
91       * Note also, that if a list of names is provided, separated by commas ({@code ","}), that
92       * this will work the same as calling this method a number of times with just one name each.
93       *
94       * @param opts The FileSystemOptions to modify.
95       * @param name resource name of additional configuration or {@code null} to unset all the
96       * values set so far.
97       * @see #getConfigNames
98       */
99      public void setConfigName(final FileSystemOptions opts, final String name)
100     {
101         if (name == null || name.isEmpty())
102         {
103             this.setParam(opts, KEY_CONFIG_NAMES, null);
104         }
105         else
106         {
107             String previousNames = this.getString(opts, KEY_CONFIG_NAMES);
108             if (previousNames == null || previousNames.isEmpty())
109             {
110                 this.setParam(opts, KEY_CONFIG_NAMES, name);
111             }
112             else
113             {
114                 this.setParam(opts, KEY_CONFIG_NAMES, previousNames + "," + name);
115             }
116         }
117     }
118 
119     /**
120      * Get paths of alternate configuration file system files.
121      *
122      * @return list of full paths of alternate configuration files or {@code null}.
123      * @param opts The FileSystemOptions.
124      * @see #setConfigPath(FileSystemOptions, Path)
125      */
126     public Path[] getConfigPaths(final FileSystemOptions opts)
127     {
128         String pathNames = this.getString(opts, KEY_CONFIG_PATHS);
129         if (pathNames == null || pathNames.isEmpty())
130         {
131             return null;
132         }
133         String[] paths = pathNames.split(",");
134         Path[] realPaths = new Path[paths.length];
135         for (int i = 0; i < paths.length; i++)
136         {
137             realPaths[i] = new Path(paths[i]);
138         }
139         return realPaths;
140     }
141 
142     /**
143      * Sets the full path of configuration file to be loaded after the defaults.
144      * <p>
145      * Specifies the path of a local file system config file to override any specific HDFS settings.
146      * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Path)}
147      * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
148      * <p>
149      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
150      * property in order to access HDFS files stored in an AWS installation (from outside their
151      * firewall). There are other possible uses too.
152      * <p>
153      * This method may be called multiple times and all the specified resources will be loaded
154      * in the order they were specified.
155      *
156      * @param opts The FileSystemOptions to modify.
157      * @param path full path of additional configuration file (local file system) or {@code null}
158      * to unset all the path values set so far.
159      */
160     public void setConfigPath(final FileSystemOptions opts, final Path path)
161     {
162         if (path == null)
163         {
164             this.setParam(opts, KEY_CONFIG_PATHS, null);
165         }
166         else
167         {
168             String previousPathNames = this.getString(opts, KEY_CONFIG_PATHS);
169             if (previousPathNames == null || previousPathNames.isEmpty())
170             {
171                 this.setParam(opts, KEY_CONFIG_PATHS, path.toString());
172             }
173             else
174             {
175                 this.setParam(opts, KEY_CONFIG_PATHS, previousPathNames + "," + path.toString());
176             }
177         }
178     }
179 
180     /**
181      * Get URLs of alternate configurations.
182      *
183      * @return list of alternate configuration URLs or {@code null}.
184      * @param opts The FileSystemOptions.
185      * @see #setConfigURL(FileSystemOptions, URL)
186      */
187     public URL[] getConfigURLs(final FileSystemOptions opts)
188     {
189         try
190         {
191             String urlNames = this.getString(opts, KEY_CONFIG_URLS);
192             if (urlNames == null || urlNames.isEmpty())
193             {
194                 return null;
195             }
196             String[] urls = urlNames.split(",");
197             URL[] realURLs = new URL[urls.length];
198             for (int i = 0; i < urls.length; i++)
199             {
200                 realURLs[i] = new URL(urls[i]);
201             }
202             return realURLs;
203         }
204         catch (MalformedURLException mue)
205         {
206             // This should never happen because we save it in the proper form
207         }
208         return null;
209     }
210 
211     /**
212      * Sets the URL of configuration file to be loaded after the defaults.
213      * <p>
214      * Specifies the URL of a config file to override any specific HDFS settings.
215      * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(URL)}
216      * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
217      * <p>
218      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
219      * property in order to access HDFS files stored in an AWS installation (from outside their
220      * firewall). There are other possible uses too.
221      * <p>
222      * This method may be called multiple times and all the specified resources will be loaded
223      * in the order they were specified.
224      *
225      * @param opts The FileSystemOptions to modify.
226      * @param url URL of additional configuration file or {@code null} to unset all the URL
227      * values set so far.
228      */
229     public void setConfigURL(final FileSystemOptions opts, final URL url)
230     {
231         if (url == null)
232         {
233             this.setParam(opts, KEY_CONFIG_URLS, null);
234         }
235         else
236         {
237             String previousURLNames = this.getString(opts, KEY_CONFIG_URLS);
238             if (previousURLNames == null || previousURLNames.isEmpty())
239             {
240                 this.setParam(opts, KEY_CONFIG_URLS, url.toString());
241             }
242             else
243             {
244                 this.setParam(opts, KEY_CONFIG_URLS, previousURLNames + "," + url.toString());
245             }
246         }
247     }
248 
249     /**
250      * Get alternate configuration input stream.
251      *
252      * @return alternate configuration input stream or {@code null}.
253      * @param opts The FileSystemOptions.
254      * @see #setConfigInputStream(FileSystemOptions, InputStream)
255      */
256     public InputStream getConfigInputStream(final FileSystemOptions opts)
257     {
258         return (InputStream)this.getParam(opts, KEY_CONFIG_STREAM);
259     }
260 
261     /**
262      * Sets the input stream of configuration file to be loaded after the defaults.
263      * <p>
264      * Specifies an input stream connected to a config file to override any specific HDFS settings.
265      * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(InputStream)}
266      * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
267      * <p>
268      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
269      * property in order to access HDFS files stored in an AWS installation (from outside their
270      * firewall). There are other possible uses too.
271      *
272      * @param opts The FileSystemOptions to modify.
273      * @param inputStream input stream of additional configuration file or {@code null} to unset
274      * the configuration input stream previously set up.
275      */
276     public void setConfigInputStream(final FileSystemOptions opts, final InputStream inputStream)
277     {
278         this.setParam(opts, KEY_CONFIG_STREAM, inputStream);
279     }
280 
281     /**
282      * Get alternate configuration object.
283      *
284      * @return alternate configuration object or {@code null}.
285      * @param opts The FileSystemOptions.
286      * @see #setConfigConfiguration(FileSystemOptions, Configuration)
287      */
288     public Configuration getConfigConfiguration(final FileSystemOptions opts)
289     {
290         return (Configuration)this.getParam(opts, KEY_CONFIG_CONF);
291     }
292 
293     /**
294      * Sets the configuration object to be loaded after the defaults.
295      * <p>
296      * Specifies an already initialized configuration object to override any specific HDFS settings.
297      * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Configuration)}
298      * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY, url)}.
299      * <p>
300      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
301      * property in order to access HDFS files stored in an AWS installation (from outside their
302      * firewall). There are other possible uses too.
303      *
304      * @param opts The FileSystemOptions to modify.
305      * @param configuration additional configuration object or {@code null} to unset any configuration
306      * object previously set.
307      */
308     public void setConfigConfiguration(final FileSystemOptions opts, final Configuration configuration)
309     {
310         this.setParam(opts, KEY_CONFIG_CONF, configuration);
311     }
312 
313 }