001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      https://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017
018package org.apache.commons.net.ftp.parser;
019
020import java.text.ParseException;
021import java.util.List;
022
023import org.apache.commons.net.ftp.Configurable;
024import org.apache.commons.net.ftp.FTPClientConfig;
025import org.apache.commons.net.ftp.FTPFile;
026import org.apache.commons.net.ftp.FTPFileEntryParser;
027
028/**
029 * Implements {@link FTPFileEntryParser} and {@link Configurable} for IBM zOS/MVS Systems.
030 *
031 * @see FTPFileEntryParser Usage instructions.
032 */
033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
034
035    static final int UNKNOWN_LIST_TYPE = -1;
036    static final int FILE_LIST_TYPE = 0;
037    static final int MEMBER_LIST_TYPE = 1;
038    static final int UNIX_LIST_TYPE = 2;
039    static final int JES_LEVEL_1_LIST_TYPE = 3;
040    static final int JES_LEVEL_2_LIST_TYPE = 4;
041
042    /**
043     * Dates are ignored for file lists, but are used for member lists where possible
044     */
045    static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
046                                                                  // 13:52
047
048    /**
049     * Matches these entries:
050     *
051     * <pre>
052     *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
053     *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
054     * </pre>
055     *
056     * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a>
057     */
058    static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
059                                                       // ignored
060            "\\S+\\s+" + // unit - ignored
061            "\\S+\\s+" + // access date - ignored
062            "\\S+\\s+" + // extents -ignored
063            // If the values are too large, the fields may be merged (NET-639)
064            "(?:\\S+\\s+)?" + // used - ignored
065            "\\S+\\s+" + // recfm - ignored
066            "\\S+\\s+" + // logical record length -ignored
067            "\\S+\\s+" + // block size - ignored
068            "(PS|PO|PO-E)\\s+" + // Dataset organization. Many exist
069            // but only support: PS, PO, PO-E
070            "(\\S+)\\s*"; // Dataset Name (file name)
071
072    /**
073     * Matches these entries:
074     *
075     * <pre>
076     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
077     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
078     * </pre>
079     */
080    static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
081            "\\S+\\s+" + // version, modification (ignored)
082            "\\S+\\s+" + // create date (ignored)
083            "(\\S+)\\s+" + // modification date
084            "(\\S+)\\s+" + // modification time
085            "\\S+\\s+" + // size in lines (ignored)
086            "\\S+\\s+" + // size in lines at creation(ignored)
087            "\\S+\\s+" + // lines modified (ignored)
088            "\\S+\\s*"; // id of user who modified (ignored)
089
090    /**
091     * Matches these entries, note: no header:
092     *
093     * <pre>
094     *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
095     *   012345678901234567890123456789012345678901234
096     *             1         2         3         4
097     * </pre>
098     */
099    static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
100            "(\\S+)\\s+" + // job number
101            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
102            "(\\S+)\\s+" + // number of spool files
103            "(\\S+)\\s+" + // Text "Spool" ignored
104            "(\\S+)\\s*" // Text "Files" ignored
105    ;
106
107    /**
108     * JES INTERFACE LEVEL 2 parser Matches these entries:
109     *
110     * <pre>
111     * JOBNAME  JOBID    OWNER    STATUS CLASS
112     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
113     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
114     * </pre>
115     *
116     * Sample output from FTP session:
117     *
118     * <pre>
119     * ftp> quote site filetype=jes
120     * 200 SITE command was accepted
121     * ftp> ls
122     * 200 Port request OK.
123     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
124     * JOBNAME  JOBID    OWNER    STATUS CLASS
125     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
126     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
127     * 250 List completed successfully.
128     * ftp> ls job01906
129     * 200 Port request OK.
130     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
131     * JOBNAME  JOBID    OWNER    STATUS CLASS
132     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
133     * --------
134     * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
135     * 001 JES2              A JESMSGLG       858
136     * 002 JES2              A JESJCL         128
137     * 003 JES2              A JESYSMSG       443
138     * 3 spool files
139     * 250 List completed successfully.
140     * </pre>
141     */
142
143    static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
144            "(\\S+)\\s+" + // job number
145            "(\\S+)\\s+" + // owner ignored
146            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
147            "(\\S+)\\s+" + // job class ignored
148            "(\\S+).*" // rest ignored
149    ;
150
151    private int isType = UNKNOWN_LIST_TYPE;
152
153    /**
154     * Fallback parser for Unix-style listings
155     */
156    private UnixFTPEntryParser unixFTPEntryParser;
157
158    /*
159     * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is
160     * the operating system on the mainframe, and is the new name for MVS)
161     *
162     * The file system on the mainframe does not have hierarchical structure as for example the Unix file system. For a more comprehensive description,
163     * please refer to the IBM manuals
164     *
165     * @LINK: https://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
166     *
167     *
168     * Dataset names =============
169     *
170     * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be
171     * max 44 characters including the dots.
172     *
173     *
174     * Dataset organization ====================
175     *
176     * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organization
177     * (DSORG). There are a number of dataset organizations, but only two are usable for FTP transfer.
178     *
179     * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset
180     *
181     * The PS file is just a flat file, as you would find it on the Unix file system.
182     *
183     * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It
184     * is possible to CD into the file, and to retrieve the individual members.
185     *
186     *
187     * Dataset record format =====================
188     *
189     * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the
190     * FTP transfer.
191     *
192     * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of
193     * records per allocated storage, and V means a variable number of records.
194     *
195     *
196     * Other notes ===========
197     *
198     * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct
199     * Access Device KJ.IOP998.ERROR.PL.UNITTEST
200     *
201     *
202     * Implementation notes ====================
203     *
204     * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed.
205     *
206     * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
207     * FTPFile.Timestamp: change time or null
208     *
209     *
210     *
211     * Additional information ======================
212     *
213     * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site
214     * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job
215     * scheduler DB2 is used to interact with a DB2 subsystem
216     *
217     * This parser supports SEQ and JES.
218     */
219
220    /**
221     * The sole constructor for a MVSFTPEntryParser object.
222     */
223    public MVSFTPEntryParser() {
224        super(""); // note the regex is set in preParse.
225        super.configure(null); // configure parser with default configurations
226    }
227
228    @Override
229    protected FTPClientConfig getDefaultConfiguration() {
230        return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null);
231    }
232
233    /**
234     * Parses entries representing a dataset list.
235     * <pre>
236     * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10
237     * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
238     * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
239     * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
240     * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU
241     * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
242     * Migrated                                                HLQ.DATASET.NAME
243     * </pre>
244     * <pre>
245     * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7]
246     * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organization. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name
247     * </pre>
248     *
249     * @param entry zosDirectoryEntry
250     * @return null: entry was not parsed.
251     */
252    private FTPFile parseFileList(final String entry) {
253        if (matches(entry)) {
254            final FTPFile file = new FTPFile();
255            file.setRawListing(entry);
256            final String name = group(2);
257            final String dsorg = group(1);
258            file.setName(name);
259
260            // DSORG
261            if ("PS".equals(dsorg)) {
262                file.setType(FTPFile.FILE_TYPE);
263            } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
264                // regex already ruled out anything other than PO or PO-E
265                file.setType(FTPFile.DIRECTORY_TYPE);
266            } else {
267                return null;
268            }
269
270            return file;
271        }
272
273        final boolean migrated = entry.startsWith("Migrated");
274        if (migrated || entry.startsWith("ARCIVE")) {
275            // Type of file is unknown for migrated datasets
276            final FTPFile file = new FTPFile();
277            file.setRawListing(entry);
278            file.setType(FTPFile.UNKNOWN_TYPE);
279            file.setName(entry.split("\\s+")[migrated ? 1 : 5]);
280            return file;
281        }
282
283        return null;
284    }
285
286    /**
287     * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an {@code FTPFile} instance. If the
288     * file listing line doesn't describe a file, then {@code null} is returned. Otherwise, a {@code FTPFile} instance representing the file is
289     * returned.
290     *
291     * @param entry A line of text from the file listing
292     * @return An FTPFile instance corresponding to the supplied entry
293     */
294    @Override
295    public FTPFile parseFTPEntry(final String entry) {
296        switch (isType) {
297        case FILE_LIST_TYPE:
298            return parseFileList(entry);
299        case MEMBER_LIST_TYPE:
300            return parseMemberList(entry);
301        case UNIX_LIST_TYPE:
302            return unixFTPEntryParser.parseFTPEntry(entry);
303        case JES_LEVEL_1_LIST_TYPE:
304            return parseJeslevel1List(entry);
305        case JES_LEVEL_2_LIST_TYPE:
306            return parseJeslevel2List(entry);
307        default:
308            break;
309        }
310
311        return null;
312    }
313
314    /**
315     * Matches these entries, note: no header:
316     *
317     * <pre>
318     * [1]      [2]      [3]   [4] [5]
319     * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
320     * 012345678901234567890123456789012345678901234
321     *           1         2         3         4
322     * -------------------------------------------
323     * Group in regex
324     * [1] Job name
325     * [2] Job number
326     * [3] Job status (INPUT,ACTIVE,OUTPUT)
327     * [4] Number of sysout files
328     * [5] The string "Spool Files"
329     * </pre>
330     *
331     * @param entry zosDirectoryEntry
332     * @return null: entry was not parsed.
333     */
334    private FTPFile parseJeslevel1List(final String entry) {
335        return parseJeslevelList(entry, 3);
336    }
337
338    /**
339     * Matches these entries:
340     *
341     * <pre>
342     * [1]      [2]      [3]     [4]    [5]
343     * JOBNAME  JOBID    OWNER   STATUS CLASS
344     * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
345     * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
346     * 012345678901234567890123456789012345678901234
347     *           1         2         3         4
348     * -------------------------------------------
349     * Group in regex
350     * [1] Job name
351     * [2] Job number
352     * [3] Owner
353     * [4] Job status (INPUT,ACTIVE,OUTPUT)
354     * [5] Job Class
355     * [6] The rest
356     * </pre>
357     *
358     * @param entry zosDirectoryEntry
359     * @return null: entry was not parsed.
360     */
361    private FTPFile parseJeslevel2List(final String entry) {
362        return parseJeslevelList(entry, 4);
363    }
364
365    private FTPFile parseJeslevelList(final String entry, final int matchNum) {
366        if (matches(entry)) {
367            final FTPFile file = new FTPFile();
368            if (group(matchNum).equalsIgnoreCase("OUTPUT")) {
369                file.setRawListing(entry);
370                final String name = group(2); /* Job Number, used by GET */
371                file.setName(name);
372                file.setType(FTPFile.FILE_TYPE);
373                return file;
374            }
375        }
376        return null;
377    }
378
379    /**
380     * Parses entries within a partitioned dataset.
381     *
382     * Format of a memberlist within a PDS:
383     *
384     * <pre>
385     *    0         1        2          3        4     5     6      7    8
386     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
387     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
388     *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
389     *
390     * -------------------------------------------
391     * [1] Name
392     * [2] VV.MM: Version . modification
393     * [3] Created: yyyy / MM / dd
394     * [4,5] Changed: yyyy / MM / dd HH:mm
395     * [6] Size: number of lines
396     * [7] Init: number of lines when first created
397     * [8] Mod: number of modified lines a last save
398     * [9] Id: User id for last update
399     * </pre>
400     *
401     * @param entry zosDirectoryEntry
402     * @return null: entry was not parsed.
403     */
404    private FTPFile parseMemberList(final String entry) {
405        final FTPFile file = new FTPFile();
406        if (matches(entry)) {
407            file.setRawListing(entry);
408            final String name = group(1);
409            final String datestr = group(2) + " " + group(3);
410            file.setName(name);
411            file.setType(FTPFile.FILE_TYPE);
412            try {
413                file.setTimestamp(super.parseTimestamp(datestr));
414            } catch (final ParseException e) {
415                // just ignore parsing errors.
416                // TODO check this is ok
417                // Drop thru to try simple parser
418            }
419            return file;
420        }
421        /*
422         * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason
423         * fails. Then just assign the name field of FTPFile.
424         */
425        if (entry != null && !entry.trim().isEmpty()) {
426            file.setRawListing(entry);
427            final String name = entry.split(" ")[0];
428            file.setName(name);
429            file.setType(FTPFile.FILE_TYPE);
430            return file;
431        }
432        return null;
433    }
434
435    /**
436     * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized:
437     * <ul>
438     *     <li>z/OS-MVS File lists,</li>
439     *     <li>z/OS-MVS Member lists,</li>
440     *     <li>Unix file lists.</li>
441     * </ul>
442     * @since 2.0
443     */
444    @Override
445    public List<String> preParse(final List<String> orig) {
446        // simply remove the header line. Composite logic will take care of the
447        // two different types of
448        // list in short order.
449        if (orig != null && !orig.isEmpty()) {
450            final String header = orig.get(0);
451            if (header.contains("Volume") && header.contains("Dsname")) {
452                setType(FILE_LIST_TYPE);
453                super.setRegex(FILE_LIST_REGEX);
454            } else if (header.contains("Name") && header.contains("Id")) {
455                setType(MEMBER_LIST_TYPE);
456                super.setRegex(MEMBER_LIST_REGEX);
457            } else if (header.startsWith("total")) {
458                setType(UNIX_LIST_TYPE);
459                unixFTPEntryParser = new UnixFTPEntryParser();
460            } else if (header.indexOf("Spool Files") >= 30) {
461                setType(JES_LEVEL_1_LIST_TYPE);
462                super.setRegex(JES_LEVEL_1_LIST_REGEX);
463            } else if (header.startsWith("JOBNAME") && header.indexOf("JOBID") > 8) { // header contains JOBNAME JOBID OWNER // STATUS CLASS
464                setType(JES_LEVEL_2_LIST_TYPE);
465                super.setRegex(JES_LEVEL_2_LIST_REGEX);
466            } else {
467                setType(UNKNOWN_LIST_TYPE);
468            }
469            if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
470                orig.remove(0);
471            }
472        }
473        return orig;
474    }
475
476    /**
477     * Sets the type of listing being processed.
478     *
479     * @param type The listing type.
480     */
481    void setType(final int type) {
482        isType = type;
483    }
484
485}