View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  package org.apache.commons.net.ftp.parser;
19  
20  import java.text.ParseException;
21  import java.util.List;
22  
23  import org.apache.commons.net.ftp.FTPClientConfig;
24  import org.apache.commons.net.ftp.FTPFile;
25  
26  /**
27   * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS
28   * Systems.
29   *
30   * @version $Id: MVSFTPEntryParser.java 1741829 2016-05-01 00:24:44Z sebb $
31   * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for
32   *      usage instructions)
33   */
34  public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
35  
36      static final int UNKNOWN_LIST_TYPE = -1;
37      static final int FILE_LIST_TYPE = 0;
38      static final int MEMBER_LIST_TYPE = 1;
39      static final int UNIX_LIST_TYPE = 2;
40      static final int JES_LEVEL_1_LIST_TYPE = 3;
41      static final int JES_LEVEL_2_LIST_TYPE = 4;
42  
43      private int isType = UNKNOWN_LIST_TYPE;
44  
45      /**
46       * Fallback parser for Unix-style listings
47       */
48      private UnixFTPEntryParser unixFTPEntryParser;
49  
50      /**
51       * Dates are ignored for file lists, but are used for member lists where
52       * possible
53       */
54      static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
55                                                                      // 13:52
56  
57      /**
58       * Matches these entries:
59       * <pre>
60       *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
61       *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
62       * </pre>
63       */
64      static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
65                                                                  // ignored
66              "\\S+\\s+" + // unit - ignored
67              "\\S+\\s+" + // access date - ignored
68              "\\S+\\s+" + // extents -ignored
69              "\\S+\\s+" + // used - ignored
70              "[FV]\\S*\\s+" + // recfm - must start with F or V
71              "\\S+\\s+" + // logical record length -ignored
72              "\\S+\\s+" + // block size - ignored
73              "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
74              // but only support: PS, PO, PO-E
75              "(\\S+)\\s*"; // Dataset Name (file name)
76  
77      /**
78       * Matches these entries:
79       * <pre>
80       *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
81       *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
82       * </pre>
83       */
84      static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
85              "\\S+\\s+" + // version, modification (ignored)
86              "\\S+\\s+" + // create date (ignored)
87              "(\\S+)\\s+" + // modification date
88              "(\\S+)\\s+" + // modification time
89              "\\S+\\s+" + // size in lines (ignored)
90              "\\S+\\s+" + // size in lines at creation(ignored)
91              "\\S+\\s+" + // lines modified (ignored)
92              "\\S+\\s*"; // id of user who modified (ignored)
93  
94      /**
95       * Matches these entries, note: no header:
96       * <pre>
97       *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
98       *   012345678901234567890123456789012345678901234
99       *             1         2         3         4
100      * </pre>
101      */
102     static final String JES_LEVEL_1_LIST_REGEX =
103             "(\\S+)\\s+" + // job name ignored
104             "(\\S+)\\s+" + // job number
105             "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
106             "(\\S+)\\s+" + // number of spool files
107             "(\\S+)\\s+" + // Text "Spool" ignored
108             "(\\S+)\\s*" // Text "Files" ignored
109     ;
110 
111     /**
112      * JES INTERFACE LEVEL 2 parser
113      * Matches these entries:
114      * <pre>
115      * JOBNAME  JOBID    OWNER    STATUS CLASS
116      * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
117      * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
118      * </pre>
119      * Sample output from FTP session:
120      * <pre>
121      * ftp> quote site filetype=jes
122      * 200 SITE command was accepted
123      * ftp> ls
124      * 200 Port request OK.
125      * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
126      * JOBNAME  JOBID    OWNER    STATUS CLASS
127      * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
128      * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
129      * 250 List completed successfully.
130      * ftp> ls job01906
131      * 200 Port request OK.
132      * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
133      * JOBNAME  JOBID    OWNER    STATUS CLASS
134      * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
135      * --------
136      * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
137      * 001 JES2              A JESMSGLG       858
138      * 002 JES2              A JESJCL         128
139      * 003 JES2              A JESYSMSG       443
140      * 3 spool files
141      * 250 List completed successfully.
142      * </pre>
143      */
144 
145     static final String JES_LEVEL_2_LIST_REGEX =
146             "(\\S+)\\s+" + // job name ignored
147             "(\\S+)\\s+" + // job number
148             "(\\S+)\\s+" + // owner ignored
149             "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
150             "(\\S+)\\s+" + // job class ignored
151             "(\\S+).*" // rest ignored
152     ;
153 
154     /*
155      * ---------------------------------------------------------------------
156      * Very brief and incomplete description of the zOS/MVS-filesystem. (Note:
157      * "zOS" is the operating system on the mainframe, and is the new name for
158      * MVS)
159      *
160      * The filesystem on the mainframe does not have hierarchal structure as for
161      * example the unix filesystem. For a more comprehensive description, please
162      * refer to the IBM manuals
163      *
164      * @LINK:
165      * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
166      *
167      *
168      * Dataset names =============
169      *
170      * A dataset name consist of a number of qualifiers separated by '.', each
171      * qualifier can be at most 8 characters, and the total length of a dataset
172      * can be max 44 characters including the dots.
173      *
174      *
175      * Dataset organisation ====================
176      *
177      * A dataset represents a piece of storage allocated on one or more disks.
178      * The structure of the storage is described with the field dataset
179      * organinsation (DSORG). There are a number of dataset organisations, but
180      * only two are usable for FTP transfer.
181      *
182      * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E:
183      * extended partitioned dataset
184      *
185      * The PS file is just a flat file, as you would find it on the unix file
186      * system.
187      *
188      * The PO and PO-E files, can be compared to a single level directory
189      * structure. A PO file consist of a number of dataset members, or files if
190      * you will. It is possible to CD into the file, and to retrieve the
191      * individual members.
192      *
193      *
194      * Dataset record format =====================
195      *
196      * The physical layout of the dataset is described on the dataset itself.
197      * There are a number of record formats (RECFM), but just a few is relavant
198      * for the FTP transfer.
199      *
200      * Any one beginning with either F or V can safely used by FTP transfer. All
201      * others should only be used with great care, so this version will just
202      * ignore the other record formats. F means a fixed number of records per
203      * allocated storage, and V means a variable number of records.
204      *
205      *
206      * Other notes ===========
207      *
208      * The file system supports automatically backup and retrieval of datasets.
209      * If a file is backed up, the ftp LIST command will return: ARCIVE Not
210      * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
211      *
212      *
213      * Implementation notes ====================
214      *
215      * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning
216      * with F or V, is fully parsed.
217      *
218      * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set.
219      * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
220      * FTPFile.Timestamp: change time or null
221      *
222      *
223      *
224      * Additional information ======================
225      *
226      * The MVS ftp server supports a number of features via the FTP interface.
227      * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2>
228      * SEQ is the default and used for normal file transfer JES is used to
229      * interact with the Job Entry Subsystem (JES) similar to a job scheduler
230      * DB2 is used to interact with a DB2 subsystem
231      *
232      * This parser supports SEQ and JES.
233      *
234      *
235      *
236      *
237      *
238      *
239      */
240 
241     /**
242      * The sole constructor for a MVSFTPEntryParser object.
243      *
244      */
245     public MVSFTPEntryParser() {
246         super(""); // note the regex is set in preParse.
247         super.configure(null); // configure parser with default configurations
248     }
249 
250     /**
251      * Parses a line of an z/OS - MVS FTP server file listing and converts it
252      * into a usable format in the form of an <code> FTPFile </code> instance.
253      * If the file listing line doesn't describe a file, then
254      * <code> null </code> is returned. Otherwise a <code> FTPFile </code>
255      * instance representing the file is returned.
256      *
257      * @param entry
258      *            A line of text from the file listing
259      * @return An FTPFile instance corresponding to the supplied entry
260      */
261     @Override
262     public FTPFile parseFTPEntry(String entry) {
263         boolean isParsed = false;
264         FTPFile f = new FTPFile();
265 
266         if (isType == FILE_LIST_TYPE) {
267             isParsed = parseFileList(f, entry);
268         } else if (isType == MEMBER_LIST_TYPE) {
269             isParsed = parseMemberList(f, entry);
270             if (!isParsed) {
271                 isParsed = parseSimpleEntry(f, entry);
272             }
273         } else if (isType == UNIX_LIST_TYPE) {
274             isParsed = parseUnixList(f, entry);
275         } else if (isType == JES_LEVEL_1_LIST_TYPE) {
276             isParsed = parseJeslevel1List(f, entry);
277         } else if (isType == JES_LEVEL_2_LIST_TYPE) {
278             isParsed = parseJeslevel2List(f, entry);
279         }
280 
281         if (!isParsed) {
282             f = null;
283         }
284 
285         return f;
286     }
287 
288     /**
289      * Parse entries representing a dataset list. Only datasets with DSORG PS or
290      * PO or PO-E and with RECFM F* or V* will be parsed.
291      *
292      * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred
293      * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80
294      * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device
295      * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO
296      * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
297      *
298      * ----------------------------------- Group within Regex [1] Volume [2]
299      * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record
300      * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg:
301      * Dataset organisation. Many exists but only support: PS, PO, PO-E [10]
302      * Dsname: Dataset name
303      *
304      * Note: When volume is ARCIVE, it means the dataset is stored somewhere in
305      * a tape archive. These entries is currently not supported by this parser.
306      * A null value is returned.
307      *
308      * @param file
309      *            will be updated with Name, Type, Timestamp if parsed.
310      * @param entry zosDirectoryEntry
311      * @return true: entry was parsed, false: entry was not parsed.
312      */
313     private boolean parseFileList(FTPFile file, String entry) {
314         if (matches(entry)) {
315             file.setRawListing(entry);
316             String name = group(2);
317             String dsorg = group(1);
318             file.setName(name);
319 
320             // DSORG
321             if ("PS".equals(dsorg)) {
322                 file.setType(FTPFile.FILE_TYPE);
323             }
324             else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
325                 // regex already ruled out anything other than PO or PO-E
326                 file.setType(FTPFile.DIRECTORY_TYPE);
327             }
328             else {
329                 return false;
330             }
331 
332             return true;
333         }
334 
335         return false;
336     }
337 
338     /**
339      * Parse entries within a partitioned dataset.
340      *
341      * Format of a memberlist within a PDS:
342      * <pre>
343      *    0         1        2          3        4     5     6      7    8
344      *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
345      *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
346      *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
347      *
348      * -------------------------------------------
349      * [1] Name
350      * [2] VV.MM: Version . modification
351      * [3] Created: yyyy / MM / dd
352      * [4,5] Changed: yyyy / MM / dd HH:mm
353      * [6] Size: number of lines
354      * [7] Init: number of lines when first created
355      * [8] Mod: number of modified lines a last save
356      * [9] Id: User id for last update
357      * </pre>
358      *
359      * @param file
360      *            will be updated with Name, Type and Timestamp if parsed.
361      * @param entry zosDirectoryEntry
362      * @return true: entry was parsed, false: entry was not parsed.
363      */
364     private boolean parseMemberList(FTPFile file, String entry) {
365         if (matches(entry)) {
366             file.setRawListing(entry);
367             String name = group(1);
368             String datestr = group(2) + " " + group(3);
369             file.setName(name);
370             file.setType(FTPFile.FILE_TYPE);
371             try {
372                 file.setTimestamp(super.parseTimestamp(datestr));
373             } catch (ParseException e) {
374                 e.printStackTrace();
375                 // just ignore parsing errors.
376                 // TODO check this is ok
377                 return false; // this is a parsing failure too.
378             }
379             return true;
380         }
381 
382         return false;
383     }
384 
385     /**
386      * Assigns the name to the first word of the entry. Only to be used from a
387      * safe context, for example from a memberlist, where the regex for some
388      * reason fails. Then just assign the name field of FTPFile.
389      *
390      * @param file
391      * @param entry
392      * @return true if the entry string is non-null and non-empty
393      */
394     private boolean parseSimpleEntry(FTPFile file, String entry) {
395         if (entry != null && entry.trim().length() > 0) {
396             file.setRawListing(entry);
397             String name = entry.split(" ")[0];
398             file.setName(name);
399             file.setType(FTPFile.FILE_TYPE);
400             return true;
401         }
402         return false;
403     }
404 
405     /**
406      * Parse the entry as a standard unix file. Using the UnixFTPEntryParser.
407      *
408      * @param file
409      * @param entry
410      * @return true: entry is parsed, false: entry could not be parsed.
411      */
412     private boolean parseUnixList(FTPFile file, String entry) {
413         file = unixFTPEntryParser.parseFTPEntry(entry);
414         if (file == null) {
415             return false;
416         }
417         return true;
418     }
419 
420     /**
421      * Matches these entries, note: no header:
422      * <pre>
423      * [1]      [2]      [3]   [4] [5]
424      * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
425      * 012345678901234567890123456789012345678901234
426      *           1         2         3         4
427      * -------------------------------------------
428      * Group in regex
429      * [1] Job name
430      * [2] Job number
431      * [3] Job status (INPUT,ACTIVE,OUTPUT)
432      * [4] Number of sysout files
433      * [5] The string "Spool Files"
434      *</pre>
435      *
436      * @param file
437      *            will be updated with Name, Type and Timestamp if parsed.
438      * @param entry zosDirectoryEntry
439      * @return true: entry was parsed, false: entry was not parsed.
440      */
441     private boolean parseJeslevel1List(FTPFile file, String entry) {
442         if (matches(entry)) {
443             if (group(3).equalsIgnoreCase("OUTPUT")) {
444                 file.setRawListing(entry);
445                 String name = group(2); /* Job Number, used by GET */
446                 file.setName(name);
447                 file.setType(FTPFile.FILE_TYPE);
448                 return true;
449             }
450         }
451 
452         return false;
453     }
454 
455     /**
456      * Matches these entries:
457      * <pre>
458      * [1]      [2]      [3]     [4]    [5]
459      * JOBNAME  JOBID    OWNER   STATUS CLASS
460      * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
461      * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
462      * 012345678901234567890123456789012345678901234
463      *           1         2         3         4
464      * -------------------------------------------
465      * Group in regex
466      * [1] Job name
467      * [2] Job number
468      * [3] Owner
469      * [4] Job status (INPUT,ACTIVE,OUTPUT)
470      * [5] Job Class
471      * [6] The rest
472      * </pre>
473      *
474      * @param file
475      *            will be updated with Name, Type and Timestamp if parsed.
476      * @param entry zosDirectoryEntry
477      * @return true: entry was parsed, false: entry was not parsed.
478      */
479     private boolean parseJeslevel2List(FTPFile file, String entry) {
480         if (matches(entry)) {
481             if (group(4).equalsIgnoreCase("OUTPUT")) {
482                 file.setRawListing(entry);
483                 String name = group(2); /* Job Number, used by GET */
484                 file.setName(name);
485                 file.setType(FTPFile.FILE_TYPE);
486                 return true;
487             }
488         }
489 
490         return false;
491     }
492 
493     /**
494      * preParse is called as part of the interface. Per definition is is called
495      * before the parsing takes place.
496      * Three kind of lists is recognize:
497      * z/OS-MVS File lists
498      * z/OS-MVS Member lists
499      * unix file lists
500      * @since 2.0
501      */
502     @Override
503     public List<String> preParse(List<String> orig) {
504         // simply remove the header line. Composite logic will take care of the
505         // two different types of
506         // list in short order.
507         if (orig != null && orig.size() > 0) {
508             String header = orig.get(0);
509             if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) {
510                 setType(FILE_LIST_TYPE);
511                 super.setRegex(FILE_LIST_REGEX);
512             } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) {
513                 setType(MEMBER_LIST_TYPE);
514                 super.setRegex(MEMBER_LIST_REGEX);
515             } else if (header.indexOf("total") == 0) {
516                 setType(UNIX_LIST_TYPE);
517                 unixFTPEntryParser = new UnixFTPEntryParser();
518             } else if (header.indexOf("Spool Files") >= 30) {
519                 setType(JES_LEVEL_1_LIST_TYPE);
520                 super.setRegex(JES_LEVEL_1_LIST_REGEX);
521             } else if (header.indexOf("JOBNAME") == 0
522                     && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
523                 setType(JES_LEVEL_2_LIST_TYPE);
524                 super.setRegex(JES_LEVEL_2_LIST_REGEX);
525             } else {
526                 setType(UNKNOWN_LIST_TYPE);
527             }
528 
529             if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
530                 orig.remove(0);
531             }
532         }
533 
534         return orig;
535     }
536 
537     /**
538      * Explicitly set the type of listing being processed.
539      * @param type The listing type.
540      */
541     void setType(int type) {
542         isType = type;
543     }
544 
545     /*
546      * @return
547      */
548     @Override
549     protected FTPClientConfig getDefaultConfiguration() {
550         return new FTPClientConfig(FTPClientConfig.SYST_MVS,
551                 DEFAULT_DATE_FORMAT, null, null, null, null);
552     }
553 
554 }