1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.commons.vfs2.provider.hdfs;
18
19 import java.io.BufferedReader;
20 import java.io.File;
21 import java.io.IOException;
22 import java.io.InputStreamReader;
23 import java.util.Map;
24
25 import org.apache.commons.io.FileUtils;
26 import org.apache.commons.lang3.SystemUtils;
27 import org.apache.commons.vfs2.FileObject;
28 import org.apache.commons.vfs2.FileSystemException;
29 import org.apache.commons.vfs2.FileType;
30 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
31 import org.apache.commons.vfs2.util.RandomAccessMode;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hdfs.DFSConfigKeys;
36 import org.apache.hadoop.hdfs.MiniDFSCluster;
37 import org.apache.log4j.Level;
38 import org.apache.log4j.Logger;
39 import org.junit.After;
40 import org.junit.AfterClass;
41 import org.junit.Assert;
42 import org.junit.BeforeClass;
43 import org.junit.Test;
44
45
46
47
48
49
50 @SuppressWarnings("resource")
51 public class HdfsFileProviderTest {
52
53
54 static {
55 System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
56 }
57
58 private static final int PORT = 8620;
59 private static final String HDFS_URI = "hdfs://localhost:" + PORT;
60 private static final String TEST_DIR1 = HDFS_URI + "/test-dir";
61 private static final Path DIR1_PATH = new Path("/test-dir");
62 private static final String TEST_FILE1 = TEST_DIR1 + "/accumulo-test-1.jar";
63 private static final Path FILE1_PATH = new Path(DIR1_PATH, "accumulo-test-1.jar");
64
65 private static DefaultFileSystemManager manager;
66 private static FileSystem hdfs;
67
68 protected static Configuration conf;
69 protected static MiniDFSCluster cluster;
70
71
72
73
74
75
76
77
78
79
80 public static void setUmask(final Configuration conf2) {
81 if (SystemUtils.IS_OS_WINDOWS) {
82 return;
83 }
84
85 try {
86 final Process p = Runtime.getRuntime().exec("/bin/sh -c umask");
87 final BufferedReader bri = new BufferedReader(new InputStreamReader(p.getInputStream()));
88 final String line = bri.readLine();
89 p.waitFor();
90 final Short umask = Short.parseShort(line.trim(), 8);
91
92
93 final int newPermission = 0777 ^ umask;
94 conf2.set("dfs.datanode.data.dir.perm", String.format("%03o", newPermission));
95 } catch (final Exception e) {
96 throw new RuntimeException("Error getting umask from O/S", e);
97 }
98 }
99
100 @BeforeClass
101 public static void setUp() throws Exception {
102 Logger.getRootLogger().setLevel(Level.ERROR);
103
104
105 final File data = new File("target/test/hdfstestdata").getAbsoluteFile();
106 data.mkdirs();
107 System.setProperty("test.build.data", data.toString());
108 FileUtils.cleanDirectory(data);
109
110
111 conf = new Configuration();
112 conf.set(FileSystem.FS_DEFAULT_NAME_KEY, HDFS_URI);
113 conf.set("hadoop.security.token.service.use_ip", "true");
114 conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024 * 1024);
115
116 setUmask(conf);
117
118 cluster = new MiniDFSCluster(PORT, conf, 1, true, true, true, null, null, null, null);
119 cluster.waitActive();
120
121
122 manager = new DefaultFileSystemManager();
123 manager.addProvider("hdfs", new HdfsFileProvider());
124 manager.init();
125 hdfs = cluster.getFileSystem();
126 }
127
128 @AfterClass
129 public static void tearDown() throws Exception {
130 if (null != hdfs) {
131 hdfs.close();
132 }
133 manager.close();
134 }
135
136 @After
137 public void after() throws Exception {
138 if (null != hdfs) {
139 hdfs.delete(DIR1_PATH, true);
140 }
141 }
142
143 private FileObject createTestFile(final FileSystem hdfs) throws IOException {
144
145 hdfs.mkdirs(DIR1_PATH);
146 final FileObject dir = manager.resolveFile(TEST_DIR1);
147 Assert.assertNotNull(dir);
148 Assert.assertTrue(dir.exists());
149 Assert.assertEquals(dir.getType(), FileType.FOLDER);
150
151
152 hdfs.create(FILE1_PATH).close();
153 final FileObject f = manager.resolveFile(TEST_FILE1);
154 Assert.assertNotNull(f);
155 Assert.assertTrue(f.exists());
156 Assert.assertEquals(f.getType(), FileType.FILE);
157 return f;
158 }
159
160 @Test
161 public void testCanRenameTo() throws Exception {
162 final FileObject fo = createTestFile(hdfs);
163 Assert.assertNotNull(fo);
164 fo.canRenameTo(fo);
165 }
166
167 @Test
168 public void testDoListChildren() throws Exception {
169 final FileObject fo = manager.resolveFile(TEST_DIR1);
170 Assert.assertNotNull(fo);
171 Assert.assertFalse(fo.exists());
172
173
174 final FileObject file = createTestFile(hdfs);
175 Assert.assertTrue(fo.exists());
176 final FileObject dir = file.getParent();
177
178 final FileObject[] children = dir.getChildren();
179 Assert.assertEquals(1, children.length);
180 Assert.assertEquals(children[0].getName(), file.getName());
181
182 }
183
184 @Test
185 public void testEquals() throws Exception {
186
187 final FileObject dir = manager.resolveFile(TEST_DIR1);
188 Assert.assertNotNull(dir);
189 Assert.assertFalse(dir.exists());
190 final FileObject file1 = createTestFile(hdfs);
191 Assert.assertTrue(file1.exists());
192 Assert.assertTrue(dir.exists());
193
194
195 final FileObject file2 = manager.resolveFile(TEST_FILE1);
196 Assert.assertEquals(file1, file2);
197
198
199 Assert.assertNotEquals(dir, file1);
200 Assert.assertNotEquals(dir, file2);
201 }
202
203 @Test
204 public void testGetAttributes() throws Exception {
205 final FileObject fo = manager.resolveFile(TEST_DIR1);
206 Assert.assertNotNull(fo);
207 Assert.assertFalse(fo.exists());
208
209
210 final FileObject file = createTestFile(hdfs);
211 Assert.assertTrue(fo.exists());
212 final Map<String, Object> attributes = file.getContent().getAttributes();
213 Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.BLOCK_SIZE.toString()));
214 Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.GROUP.toString()));
215 Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.LAST_ACCESS_TIME.toString()));
216 Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.LENGTH.toString()));
217 Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.MODIFICATION_TIME.toString()));
218 Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.OWNER.toString()));
219 Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.PERMISSIONS.toString()));
220 }
221
222 @Test
223 public void testGetContentSize() throws Exception {
224 final FileObject fo = manager.resolveFile(TEST_DIR1);
225 Assert.assertNotNull(fo);
226 Assert.assertFalse(fo.exists());
227
228
229 final FileObject file = createTestFile(hdfs);
230 Assert.assertTrue(fo.exists());
231 Assert.assertEquals(0, file.getContent().getSize());
232 Assert.assertTrue(file.getContent().isEmpty());
233 }
234
235 @Test
236 public void testGetInputStream() throws Exception {
237 final FileObject fo = manager.resolveFile(TEST_DIR1);
238 Assert.assertNotNull(fo);
239 Assert.assertFalse(fo.exists());
240
241
242 final FileObject file = createTestFile(hdfs);
243 Assert.assertTrue(fo.exists());
244 file.getContent().getInputStream().close();
245 }
246
247 @Test
248 public void testInit() throws Exception {
249 final FileObject fo = manager.resolveFile(TEST_FILE1);
250 Assert.assertNotNull(fo);
251 Assert.assertFalse(fo.exists());
252 }
253
254 @Test
255 public void testIsHidden() throws Exception {
256 final FileObject fo = manager.resolveFile(TEST_DIR1);
257 Assert.assertNotNull(fo);
258 Assert.assertFalse(fo.exists());
259
260
261 final FileObject file = createTestFile(hdfs);
262 Assert.assertTrue(fo.exists());
263 Assert.assertFalse(file.isHidden());
264 }
265
266 @Test
267 public void testIsReadable() throws Exception {
268 final FileObject fo = manager.resolveFile(TEST_DIR1);
269 Assert.assertNotNull(fo);
270 Assert.assertFalse(fo.exists());
271
272
273 final FileObject file = createTestFile(hdfs);
274 Assert.assertTrue(fo.exists());
275 Assert.assertTrue(file.isReadable());
276 }
277
278 @Test
279 public void testIsWritable() throws Exception {
280 final FileObject fo = manager.resolveFile(TEST_DIR1);
281 Assert.assertNotNull(fo);
282 Assert.assertFalse(fo.exists());
283
284
285 final FileObject file = createTestFile(hdfs);
286 Assert.assertTrue(fo.exists());
287 Assert.assertTrue(file.isWriteable());
288 }
289
290 @Test
291 public void testLastModificationTime() throws Exception {
292 final FileObject fo = manager.resolveFile(TEST_DIR1);
293 Assert.assertNotNull(fo);
294 Assert.assertFalse(fo.exists());
295
296
297 final FileObject file = createTestFile(hdfs);
298 Assert.assertTrue(fo.exists());
299 Assert.assertNotEquals(-1, file.getContent().getLastModifiedTime());
300 }
301
302 @Test(expected = FileSystemException.class)
303 public void testRandomAccessContent() throws Exception {
304 final FileObject fo = manager.resolveFile(TEST_DIR1);
305 Assert.assertNotNull(fo);
306 Assert.assertFalse(fo.exists());
307
308
309 final FileObject file = createTestFile(hdfs);
310 Assert.assertTrue(fo.exists());
311 file.getContent().getRandomAccessContent(RandomAccessMode.READWRITE).close();
312 }
313
314 @Test
315 public void testRandomAccessContent2() throws Exception {
316 final FileObject fo = manager.resolveFile(TEST_DIR1);
317 Assert.assertNotNull(fo);
318 Assert.assertFalse(fo.exists());
319
320
321 final FileObject file = createTestFile(hdfs);
322 Assert.assertTrue(fo.exists());
323 file.getContent().getRandomAccessContent(RandomAccessMode.READ).close();
324 }
325
326 }