View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  package org.apache.commons.net.ftp.parser;
19  
20  import java.text.ParseException;
21  import java.util.List;
22  
23  import org.apache.commons.net.ftp.FTPClientConfig;
24  import org.apache.commons.net.ftp.FTPFile;
25  
26  /**
27   * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS
28   * Systems.
29   *
30   * @version $Id: MVSFTPEntryParser.java 1697293 2015-08-24 01:01:00Z sebb $
31   * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for
32   *      usage instructions)
33   */
34  public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
35  
36      static final int UNKNOWN_LIST_TYPE = -1;
37      static final int FILE_LIST_TYPE = 0;
38      static final int MEMBER_LIST_TYPE = 1;
39      static final int UNIX_LIST_TYPE = 2;
40      static final int JES_LEVEL_1_LIST_TYPE = 3;
41      static final int JES_LEVEL_2_LIST_TYPE = 4;
42  
43      private int isType = UNKNOWN_LIST_TYPE;
44  
45      /**
46       * Fallback parser for Unix-style listings
47       */
48      private UnixFTPEntryParser unixFTPEntryParser;
49  
50      /**
51       * Dates are ignored for file lists, but are used for member lists where
52       * possible
53       */
54      static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
55                                                                      // 13:52
56  
57      /**
58       * Matches these entries: Volume Unit Referred Ext Used Recfm Lrecl BlkSz
59       * Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
60       *
61       */
62      static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
63                                                                  // ignored
64              "\\S+\\s+" + // unit - ignored
65              "\\S+\\s+" + // access date - ignored
66              "\\S+\\s+" + // extents -ignored
67              "\\S+\\s+" + // used - ignored
68              "[FV]\\S*\\s+" + // recfm - must start with F or V
69              "\\S+\\s+" + // logical record length -ignored
70              "\\S+\\s+" + // block size - ignored
71              "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
72              // but only support: PS, PO, PO-E
73              "(\\S+)\\s*"; // Dataset Name (file name)
74  
75      /**
76       * Matches these entries: Name VV.MM Created Changed Size Init Mod Id
77       * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001
78       */
79      static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
80              "\\S+\\s+" + // version, modification (ignored)
81              "\\S+\\s+" + // create date (ignored)
82              "(\\S+)\\s+" + // modification date
83              "(\\S+)\\s+" + // modification time
84              "\\S+\\s+" + // size in lines (ignored)
85              "\\S+\\s+" + // size in lines at creation(ignored)
86              "\\S+\\s+" + // lines modified (ignored)
87              "\\S+\\s*"; // id of user who modified (ignored)
88  
89      /**
90       * Matches these entries, note: no header: IBMUSER1 JOB01906 OUTPUT 3 Spool
91       * Files 012345678901234567890123456789012345678901234 1 2 3 4
92       */
93      static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job
94                                                                          // name
95                                                                          // ignored
96              "(\\S+)\\s+" + // job number
97              "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
98              "(\\S+)\\s+" + // number of spool files
99              "(\\S+)\\s+" + // Text "Spool" ignored
100             "(\\S+)\\s*" // Text "Files" ignored
101     ;
102 
103     /**
104      * JES INTERFACE LEVEL 2 parser Matches these entries: JOBNAME JOBID OWNER
105      * STATUS CLASS IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
106      * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
107      * 012345678901234567890123456789012345678901234 1 2 3 4
108      * 012345678901234567890123456789012345678901234567890
109      */
110 
111     static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job
112                                                                         // name
113                                                                         // ignored
114             "(\\S+)\\s+" + // job number
115             "(\\S+)\\s+" + // owner ignored
116             "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
117             "(\\S+)\\s+" + // job class ignored
118             "(\\S+).*" // rest ignored
119     ;
120 
121     /*
122      * ---------------------------------------------------------------------
123      * Very brief and incomplete description of the zOS/MVS-filesystem. (Note:
124      * "zOS" is the operating system on the mainframe, and is the new name for
125      * MVS)
126      *
127      * The filesystem on the mainframe does not have hierarchal structure as for
128      * example the unix filesystem. For a more comprehensive description, please
129      * refer to the IBM manuals
130      *
131      * @LINK:
132      * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
133      *
134      *
135      * Dataset names =============
136      *
137      * A dataset name consist of a number of qualifiers separated by '.', each
138      * qualifier can be at most 8 characters, and the total length of a dataset
139      * can be max 44 characters including the dots.
140      *
141      *
142      * Dataset organisation ====================
143      *
144      * A dataset represents a piece of storage allocated on one or more disks.
145      * The structure of the storage is described with the field dataset
146      * organinsation (DSORG). There are a number of dataset organisations, but
147      * only two are usable for FTP transfer.
148      *
149      * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E:
150      * extended partitioned dataset
151      *
152      * The PS file is just a flat file, as you would find it on the unix file
153      * system.
154      *
155      * The PO and PO-E files, can be compared to a single level directory
156      * structure. A PO file consist of a number of dataset members, or files if
157      * you will. It is possible to CD into the file, and to retrieve the
158      * individual members.
159      *
160      *
161      * Dataset record format =====================
162      *
163      * The physical layout of the dataset is described on the dataset itself.
164      * There are a number of record formats (RECFM), but just a few is relavant
165      * for the FTP transfer.
166      *
167      * Any one beginning with either F or V can safely used by FTP transfer. All
168      * others should only be used with great care, so this version will just
169      * ignore the other record formats. F means a fixed number of records per
170      * allocated storage, and V means a variable number of records.
171      *
172      *
173      * Other notes ===========
174      *
175      * The file system supports automatically backup and retrieval of datasets.
176      * If a file is backed up, the ftp LIST command will return: ARCIVE Not
177      * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
178      *
179      *
180      * Implementation notes ====================
181      *
182      * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning
183      * with F or V, is fully parsed.
184      *
185      * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set.
186      * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
187      * FTPFile.Timestamp: change time or null
188      *
189      *
190      *
191      * Additional information ======================
192      *
193      * The MVS ftp server supports a number of features via the FTP interface.
194      * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2>
195      * SEQ is the default and used for normal file transfer JES is used to
196      * interact with the Job Entry Subsystem (JES) similar to a job scheduler
197      * DB2 is used to interact with a DB2 subsystem
198      *
199      * This parser supports SEQ and JES.
200      *
201      *
202      *
203      *
204      *
205      *
206      */
207 
208     /**
209      * The sole constructor for a MVSFTPEntryParser object.
210      *
211      */
212     public MVSFTPEntryParser() {
213         super(""); // note the regex is set in preParse.
214         super.configure(null); // configure parser with default configurations
215     }
216 
217     /**
218      * Parses a line of an z/OS - MVS FTP server file listing and converts it
219      * into a usable format in the form of an <code> FTPFile </code> instance.
220      * If the file listing line doesn't describe a file, then
221      * <code> null </code> is returned. Otherwise a <code> FTPFile </code>
222      * instance representing the file is returned.
223      *
224      * @param entry
225      *            A line of text from the file listing
226      * @return An FTPFile instance corresponding to the supplied entry
227      */
228 //    @Override
229     public FTPFile parseFTPEntry(String entry) {
230         boolean isParsed = false;
231         FTPFile f = new FTPFile();
232 
233         if (isType == FILE_LIST_TYPE) {
234             isParsed = parseFileList(f, entry);
235         } else if (isType == MEMBER_LIST_TYPE) {
236             isParsed = parseMemberList(f, entry);
237             if (!isParsed) {
238                 isParsed = parseSimpleEntry(f, entry);
239             }
240         } else if (isType == UNIX_LIST_TYPE) {
241             isParsed = parseUnixList(f, entry);
242         } else if (isType == JES_LEVEL_1_LIST_TYPE) {
243             isParsed = parseJeslevel1List(f, entry);
244         } else if (isType == JES_LEVEL_2_LIST_TYPE) {
245             isParsed = parseJeslevel2List(f, entry);
246         }
247 
248         if (!isParsed) {
249             f = null;
250         }
251 
252         return f;
253     }
254 
255     /**
256      * Parse entries representing a dataset list. Only datasets with DSORG PS or
257      * PO or PO-E and with RECFM F* or V* will be parsed.
258      *
259      * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred
260      * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80
261      * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device
262      * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO
263      * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
264      *
265      * ----------------------------------- Group within Regex [1] Volume [2]
266      * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record
267      * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg:
268      * Dataset organisation. Many exists but only support: PS, PO, PO-E [10]
269      * Dsname: Dataset name
270      *
271      * Note: When volume is ARCIVE, it means the dataset is stored somewhere in
272      * a tape archive. These entries is currently not supported by this parser.
273      * A null value is returned.
274      *
275      * @param file
276      *            will be updated with Name, Type, Timestamp if parsed.
277      * @param entry zosDirectoryEntry
278      * @return true: entry was parsed, false: entry was not parsed.
279      */
280     private boolean parseFileList(FTPFile file, String entry) {
281         if (matches(entry)) {
282             file.setRawListing(entry);
283             String name = group(2);
284             String dsorg = group(1);
285             file.setName(name);
286 
287             // DSORG
288             if ("PS".equals(dsorg)) {
289                 file.setType(FTPFile.FILE_TYPE);
290             }
291             else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
292                 // regex already ruled out anything other than PO or PO-E
293                 file.setType(FTPFile.DIRECTORY_TYPE);
294             }
295             else {
296                 return false;
297             }
298 
299             return true;
300         }
301 
302         return false;
303     }
304 
305     /**
306      * Parse entries within a partitioned dataset.
307      *
308      * Format of a memberlist within a PDS: 1 2 3 4 5 6 7 8 9 Name VV.MM Created
309      * Changed Size Init Mod Id TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11
310      * 0 KIL001 TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001
311      *
312      * ------------------------------------------- [1] Name [2] VV.MM: Version .
313      * modification [3] Created: yyyy / MM / dd [4,5] Changed: yyyy / MM / dd
314      * HH:mm [6] Size: number of lines [7] Init: number of lines when first
315      * created [8] Mod: number of modified lines a last save [9] Id: User id for
316      * last update
317      *
318      *
319      * @param file
320      *            will be updated with Name, Type and Timestamp if parsed.
321      * @param entry zosDirectoryEntry
322      * @return true: entry was parsed, false: entry was not parsed.
323      */
324     private boolean parseMemberList(FTPFile file, String entry) {
325         if (matches(entry)) {
326             file.setRawListing(entry);
327             String name = group(1);
328             String datestr = group(2) + " " + group(3);
329             file.setName(name);
330             file.setType(FTPFile.FILE_TYPE);
331             try {
332                 file.setTimestamp(super.parseTimestamp(datestr));
333             } catch (ParseException e) {
334                 e.printStackTrace();
335                 // just ignore parsing errors.
336                 // TODO check this is ok
337                 return false; // this is a parsing failure too.
338             }
339             return true;
340         }
341 
342         return false;
343     }
344 
345     /**
346      * Assigns the name to the first word of the entry. Only to be used from a
347      * safe context, for example from a memberlist, where the regex for some
348      * reason fails. Then just assign the name field of FTPFile.
349      *
350      * @param file
351      * @param entry
352      * @return true if the entry string is non-null and non-empty
353      */
354     private boolean parseSimpleEntry(FTPFile file, String entry) {
355         if (entry != null && entry.trim().length() > 0) {
356             file.setRawListing(entry);
357             String name = entry.split(" ")[0];
358             file.setName(name);
359             file.setType(FTPFile.FILE_TYPE);
360             return true;
361         }
362         return false;
363     }
364 
365     /**
366      * Parse the entry as a standard unix file. Using the UnixFTPEntryParser.
367      *
368      * @param file
369      * @param entry
370      * @return true: entry is parsed, false: entry could not be parsed.
371      */
372     private boolean parseUnixList(FTPFile file, String entry) {
373         file = unixFTPEntryParser.parseFTPEntry(entry);
374         if (file == null) {
375             return false;
376         }
377         return true;
378     }
379 
380     /**
381      * Matches these entries, note: no header: [1] [2] [3] [4] [5] IBMUSER1
382      * JOB01906 OUTPUT 3 Spool Files
383      * 012345678901234567890123456789012345678901234 1 2 3 4
384      * ------------------------------------------- Group in regex [1] Job name
385      * [2] Job number [3] Job status (INPUT,ACTIVE,OUTPUT) [4] Number of sysout
386      * files [5] The string "Spool Files"
387      *
388      *
389      * @param file
390      *            will be updated with Name, Type and Timestamp if parsed.
391      * @param entry zosDirectoryEntry
392      * @return true: entry was parsed, false: entry was not parsed.
393      */
394     private boolean parseJeslevel1List(FTPFile file, String entry) {
395         if (matches(entry)) {
396             if (group(3).equalsIgnoreCase("OUTPUT")) {
397                 file.setRawListing(entry);
398                 String name = group(2); /* Job Number, used by GET */
399                 file.setName(name);
400                 file.setType(FTPFile.FILE_TYPE);
401                 return true;
402             }
403         }
404 
405         return false;
406     }
407 
408     /**
409      * Matches these entries, note: no header: [1] [2] [3] [4] [5] JOBNAME JOBID
410      * OWNER STATUS CLASS IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool
411      * files IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
412      * 012345678901234567890123456789012345678901234 1 2 3 4
413      * ------------------------------------------- Group in regex [1] Job name
414      * [2] Job number [3] Owner [4] Job status (INPUT,ACTIVE,OUTPUT) [5] Job
415      * Class [6] The rest
416      *
417      *
418      * @param file
419      *            will be updated with Name, Type and Timestamp if parsed.
420      * @param entry zosDirectoryEntry
421      * @return true: entry was parsed, false: entry was not parsed.
422      */
423     private boolean parseJeslevel2List(FTPFile file, String entry) {
424         if (matches(entry)) {
425             if (group(4).equalsIgnoreCase("OUTPUT")) {
426                 file.setRawListing(entry);
427                 String name = group(2); /* Job Number, used by GET */
428                 file.setName(name);
429                 file.setType(FTPFile.FILE_TYPE);
430                 return true;
431             }
432         }
433 
434         return false;
435     }
436 
437     /**
438      * preParse is called as part of the interface. Per definition is is called
439      * before the parsing takes place. Three kind of lists is recognize:
440      * z/OS-MVS File lists z/OS-MVS Member lists unix file lists
441      * @since 2.0
442      */
443     @Override
444     public List<String> preParse(List<String> orig) {
445         // simply remove the header line. Composite logic will take care of the
446         // two different types of
447         // list in short order.
448         if (orig != null && orig.size() > 0) {
449             String header = orig.get(0);
450             if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) {
451                 setType(FILE_LIST_TYPE);
452                 super.setRegex(FILE_LIST_REGEX);
453             } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) {
454                 setType(MEMBER_LIST_TYPE);
455                 super.setRegex(MEMBER_LIST_REGEX);
456             } else if (header.indexOf("total") == 0) {
457                 setType(UNIX_LIST_TYPE);
458                 unixFTPEntryParser = new UnixFTPEntryParser();
459             } else if (header.indexOf("Spool Files") >= 30) {
460                 setType(JES_LEVEL_1_LIST_TYPE);
461                 super.setRegex(JES_LEVEL_1_LIST_REGEX);
462             } else if (header.indexOf("JOBNAME") == 0
463                     && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
464                 setType(JES_LEVEL_2_LIST_TYPE);
465                 super.setRegex(JES_LEVEL_2_LIST_REGEX);
466             } else {
467                 setType(UNKNOWN_LIST_TYPE);
468             }
469 
470             if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
471                 orig.remove(0);
472             }
473         }
474 
475         return orig;
476     }
477 
478     /**
479      * Explicitly set the type of listing being processed.
480      * @param type The listing type.
481      */
482     void setType(int type) {
483         isType = type;
484     }
485 
486     /*
487      * @return
488      */
489     @Override
490     protected FTPClientConfig getDefaultConfiguration() {
491         return new FTPClientConfig(FTPClientConfig.SYST_MVS,
492                 DEFAULT_DATE_FORMAT, null, null, null, null);
493     }
494 
495 }