001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.net.ftp.parser; 019 020import java.text.ParseException; 021import java.util.List; 022 023import org.apache.commons.net.ftp.FTPClientConfig; 024import org.apache.commons.net.ftp.FTPFile; 025 026/** 027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS Systems. 028 * 029 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for usage instructions) 030 */ 031public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl { 032 033 static final int UNKNOWN_LIST_TYPE = -1; 034 static final int FILE_LIST_TYPE = 0; 035 static final int MEMBER_LIST_TYPE = 1; 036 static final int UNIX_LIST_TYPE = 2; 037 static final int JES_LEVEL_1_LIST_TYPE = 3; 038 static final int JES_LEVEL_2_LIST_TYPE = 4; 039 040 /** 041 * Dates are ignored for file lists, but are used for member lists where possible 042 */ 043 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18 044 // 13:52 045 046 /** 047 * Matches these entries: 048 * 049 * <pre> 050 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 051 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 052 * </pre> 053 * 054 * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a> 055 */ 056 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume 057 // ignored 058 "\\S+\\s+" + // unit - ignored 059 "\\S+\\s+" + // access date - ignored 060 "\\S+\\s+" + // extents -ignored 061 // If the values are too large, the fields may be merged (NET-639) 062 "(?:\\S+\\s+)?" + // used - ignored 063 "\\S+\\s+" + // recfm - ignored 064 "\\S+\\s+" + // logical record length -ignored 065 "\\S+\\s+" + // block size - ignored 066 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist 067 // but only support: PS, PO, PO-E 068 "(\\S+)\\s*"; // Dataset Name (file name) 069 070 /** 071 * Matches these entries: 072 * 073 * <pre> 074 * Name VV.MM Created Changed Size Init Mod Id 075 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 076 * </pre> 077 */ 078 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name 079 "\\S+\\s+" + // version, modification (ignored) 080 "\\S+\\s+" + // create date (ignored) 081 "(\\S+)\\s+" + // modification date 082 "(\\S+)\\s+" + // modification time 083 "\\S+\\s+" + // size in lines (ignored) 084 "\\S+\\s+" + // size in lines at creation(ignored) 085 "\\S+\\s+" + // lines modified (ignored) 086 "\\S+\\s*"; // id of user who modified (ignored) 087 088 /** 089 * Matches these entries, note: no header: 090 * 091 * <pre> 092 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 093 * 012345678901234567890123456789012345678901234 094 * 1 2 3 4 095 * </pre> 096 */ 097 static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored 098 "(\\S+)\\s+" + // job number 099 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) 100 "(\\S+)\\s+" + // number of spool files 101 "(\\S+)\\s+" + // Text "Spool" ignored 102 "(\\S+)\\s*" // Text "Files" ignored 103 ; 104 105 /** 106 * JES INTERFACE LEVEL 2 parser Matches these entries: 107 * 108 * <pre> 109 * JOBNAME JOBID OWNER STATUS CLASS 110 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 111 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 112 * </pre> 113 * 114 * Sample output from FTP session: 115 * 116 * <pre> 117 * ftp> quote site filetype=jes 118 * 200 SITE command was accepted 119 * ftp> ls 120 * 200 Port request OK. 121 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 122 * JOBNAME JOBID OWNER STATUS CLASS 123 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 124 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 125 * 250 List completed successfully. 126 * ftp> ls job01906 127 * 200 Port request OK. 128 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 129 * JOBNAME JOBID OWNER STATUS CLASS 130 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 131 * -------- 132 * ID STEPNAME PROCSTEP C DDNAME BYTE-COUNT 133 * 001 JES2 A JESMSGLG 858 134 * 002 JES2 A JESJCL 128 135 * 003 JES2 A JESYSMSG 443 136 * 3 spool files 137 * 250 List completed successfully. 138 * </pre> 139 */ 140 141 static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored 142 "(\\S+)\\s+" + // job number 143 "(\\S+)\\s+" + // owner ignored 144 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored 145 "(\\S+)\\s+" + // job class ignored 146 "(\\S+).*" // rest ignored 147 ; 148 149 private int isType = UNKNOWN_LIST_TYPE; 150 151 /** 152 * Fallback parser for Unix-style listings 153 */ 154 private UnixFTPEntryParser unixFTPEntryParser; 155 156 /* 157 * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is 158 * the operating system on the mainframe, and is the new name for MVS) 159 * 160 * The file system on the mainframe does not have hierarchical structure as for example the unix file system. For a more comprehensive description, 161 * please refer to the IBM manuals 162 * 163 * @LINK: http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS 164 * 165 * 166 * Dataset names ============= 167 * 168 * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be 169 * max 44 characters including the dots. 170 * 171 * 172 * Dataset organisation ==================== 173 * 174 * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organisation 175 * (DSORG). There are a number of dataset organisations, but only two are usable for FTP transfer. 176 * 177 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset 178 * 179 * The PS file is just a flat file, as you would find it on the unix file system. 180 * 181 * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It 182 * is possible to CD into the file, and to retrieve the individual members. 183 * 184 * 185 * Dataset record format ===================== 186 * 187 * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the 188 * FTP transfer. 189 * 190 * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of 191 * records per allocated storage, and V means a variable number of records. 192 * 193 * 194 * Other notes =========== 195 * 196 * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct 197 * Access Device KJ.IOP998.ERROR.PL.UNITTEST 198 * 199 * 200 * Implementation notes ==================== 201 * 202 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed. 203 * 204 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name 205 * FTPFile.Timestamp: change time or null 206 * 207 * 208 * 209 * Additional information ====================== 210 * 211 * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site 212 * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job 213 * scheduler DB2 is used to interact with a DB2 subsystem 214 * 215 * This parser supports SEQ and JES. 216 */ 217 218 /** 219 * The sole constructor for a MVSFTPEntryParser object. 220 */ 221 public MVSFTPEntryParser() { 222 super(""); // note the regex is set in preParse. 223 super.configure(null); // configure parser with default configurations 224 } 225 226 /* 227 * @return 228 */ 229 @Override 230 protected FTPClientConfig getDefaultConfiguration() { 231 return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null); 232 } 233 234 /** 235 * Parses entries representing a dataset list. 236 * <pre> 237 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 238 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 239 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 240 * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST 241 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU 242 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB 243 * Migrated HLQ.DATASET.NAME 244 * </pre> 245 * <pre> 246 * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7] 247 * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organisation. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name 248 * </pre> 249 * 250 * @param entry zosDirectoryEntry 251 * @return null: entry was not parsed. 252 */ 253 private FTPFile parseFileList(final String entry) { 254 if (matches(entry)) { 255 final FTPFile file = new FTPFile(); 256 file.setRawListing(entry); 257 final String name = group(2); 258 final String dsorg = group(1); 259 file.setName(name); 260 261 // DSORG 262 if ("PS".equals(dsorg)) { 263 file.setType(FTPFile.FILE_TYPE); 264 } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) { 265 // regex already ruled out anything other than PO or PO-E 266 file.setType(FTPFile.DIRECTORY_TYPE); 267 } else { 268 return null; 269 } 270 271 return file; 272 } 273 274 final boolean migrated = entry.startsWith("Migrated"); 275 if (migrated || entry.startsWith("ARCIVE")) { 276 // Type of file is unknown for migrated datasets 277 final FTPFile file = new FTPFile(); 278 file.setRawListing(entry); 279 file.setType(FTPFile.UNKNOWN_TYPE); 280 file.setName(entry.split("\\s+")[migrated ? 1 : 5]); 281 return file; 282 } 283 284 return null; 285 } 286 287 /** 288 * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an <code> FTPFile </code> instance. If the 289 * file listing line doesn't describe a file, then <code> null </code> is returned. Otherwise, a <code> FTPFile </code> instance representing the file is 290 * returned. 291 * 292 * @param entry A line of text from the file listing 293 * @return An FTPFile instance corresponding to the supplied entry 294 */ 295 @Override 296 public FTPFile parseFTPEntry(final String entry) { 297 switch (isType) { 298 case FILE_LIST_TYPE: 299 return parseFileList(entry); 300 case MEMBER_LIST_TYPE: 301 return parseMemberList(entry); 302 case UNIX_LIST_TYPE: 303 return unixFTPEntryParser.parseFTPEntry(entry); 304 case JES_LEVEL_1_LIST_TYPE: 305 return parseJeslevel1List(entry); 306 case JES_LEVEL_2_LIST_TYPE: 307 return parseJeslevel2List(entry); 308 default: 309 break; 310 } 311 312 return null; 313 } 314 315 /** 316 * Matches these entries, note: no header: 317 * 318 * <pre> 319 * [1] [2] [3] [4] [5] 320 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 321 * 012345678901234567890123456789012345678901234 322 * 1 2 3 4 323 * ------------------------------------------- 324 * Group in regex 325 * [1] Job name 326 * [2] Job number 327 * [3] Job status (INPUT,ACTIVE,OUTPUT) 328 * [4] Number of sysout files 329 * [5] The string "Spool Files" 330 * </pre> 331 * 332 * @param entry zosDirectoryEntry 333 * @return null: entry was not parsed. 334 */ 335 private FTPFile parseJeslevel1List(final String entry) { 336 return parseJeslevelList(entry, 3); 337 } 338 339 /** 340 * Matches these entries: 341 * 342 * <pre> 343 * [1] [2] [3] [4] [5] 344 * JOBNAME JOBID OWNER STATUS CLASS 345 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 346 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 347 * 012345678901234567890123456789012345678901234 348 * 1 2 3 4 349 * ------------------------------------------- 350 * Group in regex 351 * [1] Job name 352 * [2] Job number 353 * [3] Owner 354 * [4] Job status (INPUT,ACTIVE,OUTPUT) 355 * [5] Job Class 356 * [6] The rest 357 * </pre> 358 * 359 * @param entry zosDirectoryEntry 360 * @return null: entry was not parsed. 361 */ 362 private FTPFile parseJeslevel2List(final String entry) { 363 return parseJeslevelList(entry, 4); 364 } 365 366 private FTPFile parseJeslevelList(final String entry, final int matchNum) { 367 if (matches(entry)) { 368 final FTPFile file = new FTPFile(); 369 if (group(matchNum).equalsIgnoreCase("OUTPUT")) { 370 file.setRawListing(entry); 371 final String name = group(2); /* Job Number, used by GET */ 372 file.setName(name); 373 file.setType(FTPFile.FILE_TYPE); 374 return file; 375 } 376 } 377 return null; 378 } 379 380 /** 381 * Parses entries within a partitioned dataset. 382 * 383 * Format of a memberlist within a PDS: 384 * 385 * <pre> 386 * 0 1 2 3 4 5 6 7 8 387 * Name VV.MM Created Changed Size Init Mod Id 388 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 389 * TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001 390 * 391 * ------------------------------------------- 392 * [1] Name 393 * [2] VV.MM: Version . modification 394 * [3] Created: yyyy / MM / dd 395 * [4,5] Changed: yyyy / MM / dd HH:mm 396 * [6] Size: number of lines 397 * [7] Init: number of lines when first created 398 * [8] Mod: number of modified lines a last save 399 * [9] Id: User id for last update 400 * </pre> 401 * 402 * @param entry zosDirectoryEntry 403 * @return null: entry was not parsed. 404 */ 405 private FTPFile parseMemberList(final String entry) { 406 final FTPFile file = new FTPFile(); 407 if (matches(entry)) { 408 file.setRawListing(entry); 409 final String name = group(1); 410 final String datestr = group(2) + " " + group(3); 411 file.setName(name); 412 file.setType(FTPFile.FILE_TYPE); 413 try { 414 file.setTimestamp(super.parseTimestamp(datestr)); 415 } catch (final ParseException e) { 416 // just ignore parsing errors. 417 // TODO check this is ok 418 // Drop thru to try simple parser 419 } 420 return file; 421 } 422 423 /* 424 * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason 425 * fails. Then just assign the name field of FTPFile. 426 */ 427 if (entry != null && !entry.trim().isEmpty()) { 428 file.setRawListing(entry); 429 final String name = entry.split(" ")[0]; 430 file.setName(name); 431 file.setType(FTPFile.FILE_TYPE); 432 return file; 433 } 434 return null; 435 } 436 437 /** 438 * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized: 439 * <ul> 440 * <li>z/OS-MVS File lists,</li> 441 * <li>z/OS-MVS Member lists,</li> 442 * <li>unix file lists.</li> 443 * </ul> 444 * @since 2.0 445 */ 446 @Override 447 public List<String> preParse(final List<String> orig) { 448 // simply remove the header line. Composite logic will take care of the 449 // two different types of 450 // list in short order. 451 if (orig != null && !orig.isEmpty()) { 452 final String header = orig.get(0); 453 if (header.contains("Volume") && header.contains("Dsname")) { 454 setType(FILE_LIST_TYPE); 455 super.setRegex(FILE_LIST_REGEX); 456 } else if (header.contains("Name") && header.contains("Id")) { 457 setType(MEMBER_LIST_TYPE); 458 super.setRegex(MEMBER_LIST_REGEX); 459 } else if (header.indexOf("total") == 0) { 460 setType(UNIX_LIST_TYPE); 461 unixFTPEntryParser = new UnixFTPEntryParser(); 462 } else if (header.indexOf("Spool Files") >= 30) { 463 setType(JES_LEVEL_1_LIST_TYPE); 464 super.setRegex(JES_LEVEL_1_LIST_REGEX); 465 } else if (header.indexOf("JOBNAME") == 0 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS 466 setType(JES_LEVEL_2_LIST_TYPE); 467 super.setRegex(JES_LEVEL_2_LIST_REGEX); 468 } else { 469 setType(UNKNOWN_LIST_TYPE); 470 } 471 472 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary 473 orig.remove(0); 474 } 475 } 476 477 return orig; 478 } 479 480 /** 481 * Sets the type of listing being processed. 482 * 483 * @param type The listing type. 484 */ 485 void setType(final int type) { 486 isType = type; 487 } 488 489}