MVSFTPEntryParser.java

  1. /*
  2.  * Licensed to the Apache Software Foundation (ASF) under one or more
  3.  * contributor license agreements.  See the NOTICE file distributed with
  4.  * this work for additional information regarding copyright ownership.
  5.  * The ASF licenses this file to You under the Apache License, Version 2.0
  6.  * (the "License"); you may not use this file except in compliance with
  7.  * the License.  You may obtain a copy of the License at
  8.  *
  9.  *      http://www.apache.org/licenses/LICENSE-2.0
  10.  *
  11.  * Unless required by applicable law or agreed to in writing, software
  12.  * distributed under the License is distributed on an "AS IS" BASIS,
  13.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14.  * See the License for the specific language governing permissions and
  15.  * limitations under the License.
  16.  */

  18. package org.apache.commons.net.ftp.parser;

  20. import java.text.ParseException;
  21. import java.util.List;

  23. import org.apache.commons.net.ftp.Configurable;
  24. import org.apache.commons.net.ftp.FTPClientConfig;
  25. import org.apache.commons.net.ftp.FTPFile;
  26. import org.apache.commons.net.ftp.FTPFileEntryParser;

  28. /**
  29.  * Implements {@link FTPFileEntryParser} and {@link Configurable} for IBM zOS/MVS Systems.
  30.  *
  31.  * @see FTPFileEntryParser Usage instructions.
  32.  */
  33. public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {

  35.     static final int UNKNOWN_LIST_TYPE = -1;
  36.     static final int FILE_LIST_TYPE = 0;
  37.     static final int MEMBER_LIST_TYPE = 1;
  38.     static final int UNIX_LIST_TYPE = 2;
  39.     static final int JES_LEVEL_1_LIST_TYPE = 3;
  40.     static final int JES_LEVEL_2_LIST_TYPE = 4;

  42.     /**
  43.      * Dates are ignored for file lists, but are used for member lists where possible
  44.      */
  45.     static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
  46.                                                                   // 13:52

  48.     /**
  49.      * Matches these entries:
  50.      *
  51.      * <pre>
  52.      *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
  53.      *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
  54.      * </pre>
  55.      *
  56.      * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a>
  57.      */
  58.     static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
  59.                                                        // ignored
  60.             "\\S+\\s+" + // unit - ignored
  61.             "\\S+\\s+" + // access date - ignored
  62.             "\\S+\\s+" + // extents -ignored
  63.             // If the values are too large, the fields may be merged (NET-639)
  64.             "(?:\\S+\\s+)?" + // used - ignored
  65.             "\\S+\\s+" + // recfm - ignored
  66.             "\\S+\\s+" + // logical record length -ignored
  67.             "\\S+\\s+" + // block size - ignored
  68.             "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
  69.             // but only support: PS, PO, PO-E
  70.             "(\\S+)\\s*"; // Dataset Name (file name)

  72.     /**
  73.      * Matches these entries:
  74.      *
  75.      * <pre>
  76.      *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
  77.      *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
  78.      * </pre>
  79.      */
  80.     static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
  81.             "\\S+\\s+" + // version, modification (ignored)
  82.             "\\S+\\s+" + // create date (ignored)
  83.             "(\\S+)\\s+" + // modification date
  84.             "(\\S+)\\s+" + // modification time
  85.             "\\S+\\s+" + // size in lines (ignored)
  86.             "\\S+\\s+" + // size in lines at creation(ignored)
  87.             "\\S+\\s+" + // lines modified (ignored)
  88.             "\\S+\\s*"; // id of user who modified (ignored)

  90.     /**
  91.      * Matches these entries, note: no header:
  92.      *
  93.      * <pre>
  94.      *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
  95.      *   012345678901234567890123456789012345678901234
  96.      *             1         2         3         4
  97.      * </pre>
  98.      */
  99.     static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
  100.             "(\\S+)\\s+" + // job number
  101.             "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
  102.             "(\\S+)\\s+" + // number of spool files
  103.             "(\\S+)\\s+" + // Text "Spool" ignored
  104.             "(\\S+)\\s*" // Text "Files" ignored
  105.     ;

  107.     /**
  108.      * JES INTERFACE LEVEL 2 parser Matches these entries:
  109.      *
  110.      * <pre>
  111.      * JOBNAME  JOBID    OWNER    STATUS CLASS
  112.      * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
  113.      * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
  114.      * </pre>
  115.      *
  116.      * Sample output from FTP session:
  117.      *
  118.      * <pre>
  119.      * ftp> quote site filetype=jes
  120.      * 200 SITE command was accepted
  121.      * ftp> ls
  122.      * 200 Port request OK.
  123.      * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
  124.      * JOBNAME  JOBID    OWNER    STATUS CLASS
  125.      * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
  126.      * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
  127.      * 250 List completed successfully.
  128.      * ftp> ls job01906
  129.      * 200 Port request OK.
  130.      * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
  131.      * JOBNAME  JOBID    OWNER    STATUS CLASS
  132.      * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
  133.      * --------
  134.      * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
  135.      * 001 JES2              A JESMSGLG       858
  136.      * 002 JES2              A JESJCL         128
  137.      * 003 JES2              A JESYSMSG       443
  138.      * 3 spool files
  139.      * 250 List completed successfully.
  140.      * </pre>
  141.      */

  143.     static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
  144.             "(\\S+)\\s+" + // job number
  145.             "(\\S+)\\s+" + // owner ignored
  146.             "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
  147.             "(\\S+)\\s+" + // job class ignored
  148.             "(\\S+).*" // rest ignored
  149.     ;

  151.     private int isType = UNKNOWN_LIST_TYPE;

  153.     /**
  154.      * Fallback parser for Unix-style listings
  155.      */
  156.     private UnixFTPEntryParser unixFTPEntryParser;

  158.     /*
  159.      * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is
  160.      * the operating system on the mainframe, and is the new name for MVS)
  161.      *
  162.      * The file system on the mainframe does not have hierarchical structure as for example the unix file system. For a more comprehensive description,
  163.      * please refer to the IBM manuals
  164.      *
  165.      * @LINK: https://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
  166.      *
  167.      *
  168.      * Dataset names =============
  169.      *
  170.      * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be
  171.      * max 44 characters including the dots.
  172.      *
  173.      *
  174.      * Dataset organisation ====================
  175.      *
  176.      * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organisation
  177.      * (DSORG). There are a number of dataset organisations, but only two are usable for FTP transfer.
  178.      *
  179.      * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset
  180.      *
  181.      * The PS file is just a flat file, as you would find it on the unix file system.
  182.      *
  183.      * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It
  184.      * is possible to CD into the file, and to retrieve the individual members.
  185.      *
  186.      *
  187.      * Dataset record format =====================
  188.      *
  189.      * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the
  190.      * FTP transfer.
  191.      *
  192.      * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of
  193.      * records per allocated storage, and V means a variable number of records.
  194.      *
  195.      *
  196.      * Other notes ===========
  197.      *
  198.      * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct
  199.      * Access Device KJ.IOP998.ERROR.PL.UNITTEST
  200.      *
  201.      *
  202.      * Implementation notes ====================
  203.      *
  204.      * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed.
  205.      *
  206.      * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
  207.      * FTPFile.Timestamp: change time or null
  208.      *
  209.      *
  210.      *
  211.      * Additional information ======================
  212.      *
  213.      * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site
  214.      * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job
  215.      * scheduler DB2 is used to interact with a DB2 subsystem
  216.      *
  217.      * This parser supports SEQ and JES.
  218.      */

  220.     /**
  221.      * The sole constructor for a MVSFTPEntryParser object.
  222.      */
  223.     public MVSFTPEntryParser() {
  224.         super(""); // note the regex is set in preParse.
  225.         super.configure(null); // configure parser with default configurations
  226.     }

  228.     /*
  229.      * @return
  230.      */
  231.     @Override
  232.     protected FTPClientConfig getDefaultConfiguration() {
  233.         return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null);
  234.     }

  236.     /**
  237.      * Parses entries representing a dataset list.
  238.      * <pre>
  239.      * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10
  240.      * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
  241.      * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
  242.      * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
  243.      * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU
  244.      * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
  245.      * Migrated                                                HLQ.DATASET.NAME
  246.      * </pre>
  247.      * <pre>
  248.      * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7]
  249.      * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organisation. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name
  250.      * </pre>
  251.      *
  252.      * @param entry zosDirectoryEntry
  253.      * @return null: entry was not parsed.
  254.      */
  255.     private FTPFile parseFileList(final String entry) {
  256.         if (matches(entry)) {
  257.             final FTPFile file = new FTPFile();
  258.             file.setRawListing(entry);
  259.             final String name = group(2);
  260.             final String dsorg = group(1);
  261.             file.setName(name);

  263.             // DSORG
  264.             if ("PS".equals(dsorg)) {
  265.                 file.setType(FTPFile.FILE_TYPE);
  266.             } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
  267.                 // regex already ruled out anything other than PO or PO-E
  268.                 file.setType(FTPFile.DIRECTORY_TYPE);
  269.             } else {
  270.                 return null;
  271.             }

  273.             return file;
  274.         }

  276.         final boolean migrated = entry.startsWith("Migrated");
  277.         if (migrated || entry.startsWith("ARCIVE")) {
  278.             // Type of file is unknown for migrated datasets
  279.             final FTPFile file = new FTPFile();
  280.             file.setRawListing(entry);
  281.             file.setType(FTPFile.UNKNOWN_TYPE);
  282.             file.setName(entry.split("\\s+")[migrated ? 1 : 5]);
  283.             return file;
  284.         }

  286.         return null;
  287.     }

  289.     /**
  290.      * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an <code>FTPFile</code> instance. If the
  291.      * file listing line doesn't describe a file, then <code>null</code> is returned. Otherwise, a <code>FTPFile</code> instance representing the file is
  292.      * returned.
  293.      *
  294.      * @param entry A line of text from the file listing
  295.      * @return An FTPFile instance corresponding to the supplied entry
  296.      */
  297.     @Override
  298.     public FTPFile parseFTPEntry(final String entry) {
  299.         switch (isType) {
  300.         case FILE_LIST_TYPE:
  301.             return parseFileList(entry);
  302.         case MEMBER_LIST_TYPE:
  303.             return parseMemberList(entry);
  304.         case UNIX_LIST_TYPE:
  305.             return unixFTPEntryParser.parseFTPEntry(entry);
  306.         case JES_LEVEL_1_LIST_TYPE:
  307.             return parseJeslevel1List(entry);
  308.         case JES_LEVEL_2_LIST_TYPE:
  309.             return parseJeslevel2List(entry);
  310.         default:
  311.             break;
  312.         }

  314.         return null;
  315.     }

  317.     /**
  318.      * Matches these entries, note: no header:
  319.      *
  320.      * <pre>
  321.      * [1]      [2]      [3]   [4] [5]
  322.      * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
  323.      * 012345678901234567890123456789012345678901234
  324.      *           1         2         3         4
  325.      * -------------------------------------------
  326.      * Group in regex
  327.      * [1] Job name
  328.      * [2] Job number
  329.      * [3] Job status (INPUT,ACTIVE,OUTPUT)
  330.      * [4] Number of sysout files
  331.      * [5] The string "Spool Files"
  332.      * </pre>
  333.      *
  334.      * @param entry zosDirectoryEntry
  335.      * @return null: entry was not parsed.
  336.      */
  337.     private FTPFile parseJeslevel1List(final String entry) {
  338.         return parseJeslevelList(entry, 3);
  339.     }

  341.     /**
  342.      * Matches these entries:
  343.      *
  344.      * <pre>
  345.      * [1]      [2]      [3]     [4]    [5]
  346.      * JOBNAME  JOBID    OWNER   STATUS CLASS
  347.      * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
  348.      * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
  349.      * 012345678901234567890123456789012345678901234
  350.      *           1         2         3         4
  351.      * -------------------------------------------
  352.      * Group in regex
  353.      * [1] Job name
  354.      * [2] Job number
  355.      * [3] Owner
  356.      * [4] Job status (INPUT,ACTIVE,OUTPUT)
  357.      * [5] Job Class
  358.      * [6] The rest
  359.      * </pre>
  360.      *
  361.      * @param entry zosDirectoryEntry
  362.      * @return null: entry was not parsed.
  363.      */
  364.     private FTPFile parseJeslevel2List(final String entry) {
  365.         return parseJeslevelList(entry, 4);
  366.     }

  368.     private FTPFile parseJeslevelList(final String entry, final int matchNum) {
  369.         if (matches(entry)) {
  370.             final FTPFile file = new FTPFile();
  371.             if (group(matchNum).equalsIgnoreCase("OUTPUT")) {
  372.                 file.setRawListing(entry);
  373.                 final String name = group(2); /* Job Number, used by GET */
  374.                 file.setName(name);
  375.                 file.setType(FTPFile.FILE_TYPE);
  376.                 return file;
  377.             }
  378.         }
  379.         return null;
  380.     }

  382.     /**
  383.      * Parses entries within a partitioned dataset.
  384.      *
  385.      * Format of a memberlist within a PDS:
  386.      *
  387.      * <pre>
  388.      *    0         1        2          3        4     5     6      7    8
  389.      *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
  390.      *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
  391.      *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
  392.      *
  393.      * -------------------------------------------
  394.      * [1] Name
  395.      * [2] VV.MM: Version . modification
  396.      * [3] Created: yyyy / MM / dd
  397.      * [4,5] Changed: yyyy / MM / dd HH:mm
  398.      * [6] Size: number of lines
  399.      * [7] Init: number of lines when first created
  400.      * [8] Mod: number of modified lines a last save
  401.      * [9] Id: User id for last update
  402.      * </pre>
  403.      *
  404.      * @param entry zosDirectoryEntry
  405.      * @return null: entry was not parsed.
  406.      */
  407.     private FTPFile parseMemberList(final String entry) {
  408.         final FTPFile file = new FTPFile();
  409.         if (matches(entry)) {
  410.             file.setRawListing(entry);
  411.             final String name = group(1);
  412.             final String datestr = group(2) + " " + group(3);
  413.             file.setName(name);
  414.             file.setType(FTPFile.FILE_TYPE);
  415.             try {
  416.                 file.setTimestamp(super.parseTimestamp(datestr));
  417.             } catch (final ParseException e) {
  418.                 // just ignore parsing errors.
  419.                 // TODO check this is ok
  420.                 // Drop thru to try simple parser
  421.             }
  422.             return file;
  423.         }

  425.         /*
  426.          * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason
  427.          * fails. Then just assign the name field of FTPFile.
  428.          */
  429.         if (entry != null && !entry.trim().isEmpty()) {
  430.             file.setRawListing(entry);
  431.             final String name = entry.split(" ")[0];
  432.             file.setName(name);
  433.             file.setType(FTPFile.FILE_TYPE);
  434.             return file;
  435.         }
  436.         return null;
  437.     }

  439.     /**
  440.      * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized:
  441.      * <ul>
  442.      *     <li>z/OS-MVS File lists,</li>
  443.      *     <li>z/OS-MVS Member lists,</li>
  444.      *     <li>unix file lists.</li>
  445.      * </ul>
  446.      * @since 2.0
  447.      */
  448.     @Override
  449.     public List<String> preParse(final List<String> orig) {
  450.         // simply remove the header line. Composite logic will take care of the
  451.         // two different types of
  452.         // list in short order.
  453.         if (orig != null && !orig.isEmpty()) {
  454.             final String header = orig.get(0);
  455.             if (header.contains("Volume") && header.contains("Dsname")) {
  456.                 setType(FILE_LIST_TYPE);
  457.                 super.setRegex(FILE_LIST_REGEX);
  458.             } else if (header.contains("Name") && header.contains("Id")) {
  459.                 setType(MEMBER_LIST_TYPE);
  460.                 super.setRegex(MEMBER_LIST_REGEX);
  461.             } else if (header.startsWith("total")) {
  462.                 setType(UNIX_LIST_TYPE);
  463.                 unixFTPEntryParser = new UnixFTPEntryParser();
  464.             } else if (header.indexOf("Spool Files") >= 30) {
  465.                 setType(JES_LEVEL_1_LIST_TYPE);
  466.                 super.setRegex(JES_LEVEL_1_LIST_REGEX);
  467.             } else if (header.startsWith("JOBNAME") && header.indexOf("JOBID") > 8) { // header contains JOBNAME JOBID OWNER // STATUS CLASS
  468.                 setType(JES_LEVEL_2_LIST_TYPE);
  469.                 super.setRegex(JES_LEVEL_2_LIST_REGEX);
  470.             } else {
  471.                 setType(UNKNOWN_LIST_TYPE);
  472.             }

  474.             if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
  475.                 orig.remove(0);
  476.             }
  477.         }

  479.         return orig;
  480.     }

  482.     /**
  483.      * Sets the type of listing being processed.
  484.      *
  485.      * @param type The listing type.
  486.      */
  487.     void setType(final int type) {
  488.         isType = type;
  489.     }

  491. }
Created with JaCoCo 0.8.12.202403310830