001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * https://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.net.ftp.parser; 019 020import java.text.ParseException; 021import java.util.List; 022 023import org.apache.commons.net.ftp.Configurable; 024import org.apache.commons.net.ftp.FTPClientConfig; 025import org.apache.commons.net.ftp.FTPFile; 026import org.apache.commons.net.ftp.FTPFileEntryParser; 027 028/** 029 * Implements {@link FTPFileEntryParser} and {@link Configurable} for IBM zOS/MVS Systems. 030 * 031 * @see FTPFileEntryParser Usage instructions. 032 */ 033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl { 034 035 static final int UNKNOWN_LIST_TYPE = -1; 036 static final int FILE_LIST_TYPE = 0; 037 static final int MEMBER_LIST_TYPE = 1; 038 static final int UNIX_LIST_TYPE = 2; 039 static final int JES_LEVEL_1_LIST_TYPE = 3; 040 static final int JES_LEVEL_2_LIST_TYPE = 4; 041 042 /** 043 * Dates are ignored for file lists, but are used for member lists where possible 044 */ 045 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18 046 // 13:52 047 048 /** 049 * Matches these entries: 050 * 051 * <pre> 052 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 053 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 054 * </pre> 055 * 056 * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a> 057 */ 058 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume 059 // ignored 060 "\\S+\\s+" + // unit - ignored 061 "\\S+\\s+" + // access date - ignored 062 "\\S+\\s+" + // extents -ignored 063 // If the values are too large, the fields may be merged (NET-639) 064 "(?:\\S+\\s+)?" + // used - ignored 065 "\\S+\\s+" + // recfm - ignored 066 "\\S+\\s+" + // logical record length -ignored 067 "\\S+\\s+" + // block size - ignored 068 "(PS|PO|PO-E)\\s+" + // Dataset organization. Many exist 069 // but only support: PS, PO, PO-E 070 "(\\S+)\\s*"; // Dataset Name (file name) 071 072 /** 073 * Matches these entries: 074 * 075 * <pre> 076 * Name VV.MM Created Changed Size Init Mod Id 077 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 078 * </pre> 079 */ 080 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name 081 "\\S+\\s+" + // version, modification (ignored) 082 "\\S+\\s+" + // create date (ignored) 083 "(\\S+)\\s+" + // modification date 084 "(\\S+)\\s+" + // modification time 085 "\\S+\\s+" + // size in lines (ignored) 086 "\\S+\\s+" + // size in lines at creation(ignored) 087 "\\S+\\s+" + // lines modified (ignored) 088 "\\S+\\s*"; // id of user who modified (ignored) 089 090 /** 091 * Matches these entries, note: no header: 092 * 093 * <pre> 094 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 095 * 012345678901234567890123456789012345678901234 096 * 1 2 3 4 097 * </pre> 098 */ 099 static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored 100 "(\\S+)\\s+" + // job number 101 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) 102 "(\\S+)\\s+" + // number of spool files 103 "(\\S+)\\s+" + // Text "Spool" ignored 104 "(\\S+)\\s*" // Text "Files" ignored 105 ; 106 107 /** 108 * JES INTERFACE LEVEL 2 parser Matches these entries: 109 * 110 * <pre> 111 * JOBNAME JOBID OWNER STATUS CLASS 112 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 113 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 114 * </pre> 115 * 116 * Sample output from FTP session: 117 * 118 * <pre> 119 * ftp> quote site filetype=jes 120 * 200 SITE command was accepted 121 * ftp> ls 122 * 200 Port request OK. 123 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 124 * JOBNAME JOBID OWNER STATUS CLASS 125 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 126 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 127 * 250 List completed successfully. 128 * ftp> ls job01906 129 * 200 Port request OK. 130 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 131 * JOBNAME JOBID OWNER STATUS CLASS 132 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 133 * -------- 134 * ID STEPNAME PROCSTEP C DDNAME BYTE-COUNT 135 * 001 JES2 A JESMSGLG 858 136 * 002 JES2 A JESJCL 128 137 * 003 JES2 A JESYSMSG 443 138 * 3 spool files 139 * 250 List completed successfully. 140 * </pre> 141 */ 142 143 static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored 144 "(\\S+)\\s+" + // job number 145 "(\\S+)\\s+" + // owner ignored 146 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored 147 "(\\S+)\\s+" + // job class ignored 148 "(\\S+).*" // rest ignored 149 ; 150 151 private int isType = UNKNOWN_LIST_TYPE; 152 153 /** 154 * Fallback parser for Unix-style listings 155 */ 156 private UnixFTPEntryParser unixFTPEntryParser; 157 158 /* 159 * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is 160 * the operating system on the mainframe, and is the new name for MVS) 161 * 162 * The file system on the mainframe does not have hierarchical structure as for example the Unix file system. For a more comprehensive description, 163 * please refer to the IBM manuals 164 * 165 * @LINK: https://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS 166 * 167 * 168 * Dataset names ============= 169 * 170 * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be 171 * max 44 characters including the dots. 172 * 173 * 174 * Dataset organization ==================== 175 * 176 * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organization 177 * (DSORG). There are a number of dataset organizations, but only two are usable for FTP transfer. 178 * 179 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset 180 * 181 * The PS file is just a flat file, as you would find it on the Unix file system. 182 * 183 * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It 184 * is possible to CD into the file, and to retrieve the individual members. 185 * 186 * 187 * Dataset record format ===================== 188 * 189 * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the 190 * FTP transfer. 191 * 192 * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of 193 * records per allocated storage, and V means a variable number of records. 194 * 195 * 196 * Other notes =========== 197 * 198 * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct 199 * Access Device KJ.IOP998.ERROR.PL.UNITTEST 200 * 201 * 202 * Implementation notes ==================== 203 * 204 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed. 205 * 206 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name 207 * FTPFile.Timestamp: change time or null 208 * 209 * 210 * 211 * Additional information ====================== 212 * 213 * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site 214 * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job 215 * scheduler DB2 is used to interact with a DB2 subsystem 216 * 217 * This parser supports SEQ and JES. 218 */ 219 220 /** 221 * The sole constructor for a MVSFTPEntryParser object. 222 */ 223 public MVSFTPEntryParser() { 224 super(""); // note the regex is set in preParse. 225 super.configure(null); // configure parser with default configurations 226 } 227 228 @Override 229 protected FTPClientConfig getDefaultConfiguration() { 230 return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null); 231 } 232 233 /** 234 * Parses entries representing a dataset list. 235 * <pre> 236 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 237 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 238 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 239 * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST 240 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU 241 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB 242 * Migrated HLQ.DATASET.NAME 243 * </pre> 244 * <pre> 245 * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7] 246 * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organization. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name 247 * </pre> 248 * 249 * @param entry zosDirectoryEntry 250 * @return null: entry was not parsed. 251 */ 252 private FTPFile parseFileList(final String entry) { 253 if (matches(entry)) { 254 final FTPFile file = new FTPFile(); 255 file.setRawListing(entry); 256 final String name = group(2); 257 final String dsorg = group(1); 258 file.setName(name); 259 260 // DSORG 261 if ("PS".equals(dsorg)) { 262 file.setType(FTPFile.FILE_TYPE); 263 } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) { 264 // regex already ruled out anything other than PO or PO-E 265 file.setType(FTPFile.DIRECTORY_TYPE); 266 } else { 267 return null; 268 } 269 270 return file; 271 } 272 273 final boolean migrated = entry.startsWith("Migrated"); 274 if (migrated || entry.startsWith("ARCIVE")) { 275 // Type of file is unknown for migrated datasets 276 final FTPFile file = new FTPFile(); 277 file.setRawListing(entry); 278 file.setType(FTPFile.UNKNOWN_TYPE); 279 file.setName(entry.split("\\s+")[migrated ? 1 : 5]); 280 return file; 281 } 282 283 return null; 284 } 285 286 /** 287 * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an {@code FTPFile} instance. If the 288 * file listing line doesn't describe a file, then {@code null} is returned. Otherwise, a {@code FTPFile} instance representing the file is 289 * returned. 290 * 291 * @param entry A line of text from the file listing 292 * @return An FTPFile instance corresponding to the supplied entry 293 */ 294 @Override 295 public FTPFile parseFTPEntry(final String entry) { 296 switch (isType) { 297 case FILE_LIST_TYPE: 298 return parseFileList(entry); 299 case MEMBER_LIST_TYPE: 300 return parseMemberList(entry); 301 case UNIX_LIST_TYPE: 302 return unixFTPEntryParser.parseFTPEntry(entry); 303 case JES_LEVEL_1_LIST_TYPE: 304 return parseJeslevel1List(entry); 305 case JES_LEVEL_2_LIST_TYPE: 306 return parseJeslevel2List(entry); 307 default: 308 break; 309 } 310 311 return null; 312 } 313 314 /** 315 * Matches these entries, note: no header: 316 * 317 * <pre> 318 * [1] [2] [3] [4] [5] 319 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 320 * 012345678901234567890123456789012345678901234 321 * 1 2 3 4 322 * ------------------------------------------- 323 * Group in regex 324 * [1] Job name 325 * [2] Job number 326 * [3] Job status (INPUT,ACTIVE,OUTPUT) 327 * [4] Number of sysout files 328 * [5] The string "Spool Files" 329 * </pre> 330 * 331 * @param entry zosDirectoryEntry 332 * @return null: entry was not parsed. 333 */ 334 private FTPFile parseJeslevel1List(final String entry) { 335 return parseJeslevelList(entry, 3); 336 } 337 338 /** 339 * Matches these entries: 340 * 341 * <pre> 342 * [1] [2] [3] [4] [5] 343 * JOBNAME JOBID OWNER STATUS CLASS 344 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 345 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 346 * 012345678901234567890123456789012345678901234 347 * 1 2 3 4 348 * ------------------------------------------- 349 * Group in regex 350 * [1] Job name 351 * [2] Job number 352 * [3] Owner 353 * [4] Job status (INPUT,ACTIVE,OUTPUT) 354 * [5] Job Class 355 * [6] The rest 356 * </pre> 357 * 358 * @param entry zosDirectoryEntry 359 * @return null: entry was not parsed. 360 */ 361 private FTPFile parseJeslevel2List(final String entry) { 362 return parseJeslevelList(entry, 4); 363 } 364 365 private FTPFile parseJeslevelList(final String entry, final int matchNum) { 366 if (matches(entry)) { 367 final FTPFile file = new FTPFile(); 368 if (group(matchNum).equalsIgnoreCase("OUTPUT")) { 369 file.setRawListing(entry); 370 final String name = group(2); /* Job Number, used by GET */ 371 file.setName(name); 372 file.setType(FTPFile.FILE_TYPE); 373 return file; 374 } 375 } 376 return null; 377 } 378 379 /** 380 * Parses entries within a partitioned dataset. 381 * 382 * Format of a memberlist within a PDS: 383 * 384 * <pre> 385 * 0 1 2 3 4 5 6 7 8 386 * Name VV.MM Created Changed Size Init Mod Id 387 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 388 * TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001 389 * 390 * ------------------------------------------- 391 * [1] Name 392 * [2] VV.MM: Version . modification 393 * [3] Created: yyyy / MM / dd 394 * [4,5] Changed: yyyy / MM / dd HH:mm 395 * [6] Size: number of lines 396 * [7] Init: number of lines when first created 397 * [8] Mod: number of modified lines a last save 398 * [9] Id: User id for last update 399 * </pre> 400 * 401 * @param entry zosDirectoryEntry 402 * @return null: entry was not parsed. 403 */ 404 private FTPFile parseMemberList(final String entry) { 405 final FTPFile file = new FTPFile(); 406 if (matches(entry)) { 407 file.setRawListing(entry); 408 final String name = group(1); 409 final String datestr = group(2) + " " + group(3); 410 file.setName(name); 411 file.setType(FTPFile.FILE_TYPE); 412 try { 413 file.setTimestamp(super.parseTimestamp(datestr)); 414 } catch (final ParseException e) { 415 // just ignore parsing errors. 416 // TODO check this is ok 417 // Drop thru to try simple parser 418 } 419 return file; 420 } 421 /* 422 * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason 423 * fails. Then just assign the name field of FTPFile. 424 */ 425 if (entry != null && !entry.trim().isEmpty()) { 426 file.setRawListing(entry); 427 final String name = entry.split(" ")[0]; 428 file.setName(name); 429 file.setType(FTPFile.FILE_TYPE); 430 return file; 431 } 432 return null; 433 } 434 435 /** 436 * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized: 437 * <ul> 438 * <li>z/OS-MVS File lists,</li> 439 * <li>z/OS-MVS Member lists,</li> 440 * <li>Unix file lists.</li> 441 * </ul> 442 * @since 2.0 443 */ 444 @Override 445 public List<String> preParse(final List<String> orig) { 446 // simply remove the header line. Composite logic will take care of the 447 // two different types of 448 // list in short order. 449 if (orig != null && !orig.isEmpty()) { 450 final String header = orig.get(0); 451 if (header.contains("Volume") && header.contains("Dsname")) { 452 setType(FILE_LIST_TYPE); 453 super.setRegex(FILE_LIST_REGEX); 454 } else if (header.contains("Name") && header.contains("Id")) { 455 setType(MEMBER_LIST_TYPE); 456 super.setRegex(MEMBER_LIST_REGEX); 457 } else if (header.startsWith("total")) { 458 setType(UNIX_LIST_TYPE); 459 unixFTPEntryParser = new UnixFTPEntryParser(); 460 } else if (header.indexOf("Spool Files") >= 30) { 461 setType(JES_LEVEL_1_LIST_TYPE); 462 super.setRegex(JES_LEVEL_1_LIST_REGEX); 463 } else if (header.startsWith("JOBNAME") && header.indexOf("JOBID") > 8) { // header contains JOBNAME JOBID OWNER // STATUS CLASS 464 setType(JES_LEVEL_2_LIST_TYPE); 465 super.setRegex(JES_LEVEL_2_LIST_REGEX); 466 } else { 467 setType(UNKNOWN_LIST_TYPE); 468 } 469 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary 470 orig.remove(0); 471 } 472 } 473 return orig; 474 } 475 476 /** 477 * Sets the type of listing being processed. 478 * 479 * @param type The listing type. 480 */ 481 void setType(final int type) { 482 isType = type; 483 } 484 485}