001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.net.ftp.parser; 019 020import java.text.ParseException; 021import java.util.List; 022 023import org.apache.commons.net.ftp.FTPClientConfig; 024import org.apache.commons.net.ftp.FTPFile; 025 026/** 027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS 028 * Systems. 029 * 030 * @version $Id: MVSFTPEntryParser.java 1741829 2016-05-01 00:24:44Z sebb $ 031 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for 032 * usage instructions) 033 */ 034public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl { 035 036 static final int UNKNOWN_LIST_TYPE = -1; 037 static final int FILE_LIST_TYPE = 0; 038 static final int MEMBER_LIST_TYPE = 1; 039 static final int UNIX_LIST_TYPE = 2; 040 static final int JES_LEVEL_1_LIST_TYPE = 3; 041 static final int JES_LEVEL_2_LIST_TYPE = 4; 042 043 private int isType = UNKNOWN_LIST_TYPE; 044 045 /** 046 * Fallback parser for Unix-style listings 047 */ 048 private UnixFTPEntryParser unixFTPEntryParser; 049 050 /** 051 * Dates are ignored for file lists, but are used for member lists where 052 * possible 053 */ 054 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18 055 // 13:52 056 057 /** 058 * Matches these entries: 059 * <pre> 060 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 061 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 062 * </pre> 063 */ 064 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume 065 // ignored 066 "\\S+\\s+" + // unit - ignored 067 "\\S+\\s+" + // access date - ignored 068 "\\S+\\s+" + // extents -ignored 069 "\\S+\\s+" + // used - ignored 070 "[FV]\\S*\\s+" + // recfm - must start with F or V 071 "\\S+\\s+" + // logical record length -ignored 072 "\\S+\\s+" + // block size - ignored 073 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist 074 // but only support: PS, PO, PO-E 075 "(\\S+)\\s*"; // Dataset Name (file name) 076 077 /** 078 * Matches these entries: 079 * <pre> 080 * Name VV.MM Created Changed Size Init Mod Id 081 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 082 * </pre> 083 */ 084 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name 085 "\\S+\\s+" + // version, modification (ignored) 086 "\\S+\\s+" + // create date (ignored) 087 "(\\S+)\\s+" + // modification date 088 "(\\S+)\\s+" + // modification time 089 "\\S+\\s+" + // size in lines (ignored) 090 "\\S+\\s+" + // size in lines at creation(ignored) 091 "\\S+\\s+" + // lines modified (ignored) 092 "\\S+\\s*"; // id of user who modified (ignored) 093 094 /** 095 * Matches these entries, note: no header: 096 * <pre> 097 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 098 * 012345678901234567890123456789012345678901234 099 * 1 2 3 4 100 * </pre> 101 */ 102 static final String JES_LEVEL_1_LIST_REGEX = 103 "(\\S+)\\s+" + // job name ignored 104 "(\\S+)\\s+" + // job number 105 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) 106 "(\\S+)\\s+" + // number of spool files 107 "(\\S+)\\s+" + // Text "Spool" ignored 108 "(\\S+)\\s*" // Text "Files" ignored 109 ; 110 111 /** 112 * JES INTERFACE LEVEL 2 parser 113 * Matches these entries: 114 * <pre> 115 * JOBNAME JOBID OWNER STATUS CLASS 116 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 117 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 118 * </pre> 119 * Sample output from FTP session: 120 * <pre> 121 * ftp> quote site filetype=jes 122 * 200 SITE command was accepted 123 * ftp> ls 124 * 200 Port request OK. 125 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 126 * JOBNAME JOBID OWNER STATUS CLASS 127 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 128 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 129 * 250 List completed successfully. 130 * ftp> ls job01906 131 * 200 Port request OK. 132 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 133 * JOBNAME JOBID OWNER STATUS CLASS 134 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 135 * -------- 136 * ID STEPNAME PROCSTEP C DDNAME BYTE-COUNT 137 * 001 JES2 A JESMSGLG 858 138 * 002 JES2 A JESJCL 128 139 * 003 JES2 A JESYSMSG 443 140 * 3 spool files 141 * 250 List completed successfully. 142 * </pre> 143 */ 144 145 static final String JES_LEVEL_2_LIST_REGEX = 146 "(\\S+)\\s+" + // job name ignored 147 "(\\S+)\\s+" + // job number 148 "(\\S+)\\s+" + // owner ignored 149 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored 150 "(\\S+)\\s+" + // job class ignored 151 "(\\S+).*" // rest ignored 152 ; 153 154 /* 155 * --------------------------------------------------------------------- 156 * Very brief and incomplete description of the zOS/MVS-filesystem. (Note: 157 * "zOS" is the operating system on the mainframe, and is the new name for 158 * MVS) 159 * 160 * The filesystem on the mainframe does not have hierarchal structure as for 161 * example the unix filesystem. For a more comprehensive description, please 162 * refer to the IBM manuals 163 * 164 * @LINK: 165 * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS 166 * 167 * 168 * Dataset names ============= 169 * 170 * A dataset name consist of a number of qualifiers separated by '.', each 171 * qualifier can be at most 8 characters, and the total length of a dataset 172 * can be max 44 characters including the dots. 173 * 174 * 175 * Dataset organisation ==================== 176 * 177 * A dataset represents a piece of storage allocated on one or more disks. 178 * The structure of the storage is described with the field dataset 179 * organinsation (DSORG). There are a number of dataset organisations, but 180 * only two are usable for FTP transfer. 181 * 182 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: 183 * extended partitioned dataset 184 * 185 * The PS file is just a flat file, as you would find it on the unix file 186 * system. 187 * 188 * The PO and PO-E files, can be compared to a single level directory 189 * structure. A PO file consist of a number of dataset members, or files if 190 * you will. It is possible to CD into the file, and to retrieve the 191 * individual members. 192 * 193 * 194 * Dataset record format ===================== 195 * 196 * The physical layout of the dataset is described on the dataset itself. 197 * There are a number of record formats (RECFM), but just a few is relavant 198 * for the FTP transfer. 199 * 200 * Any one beginning with either F or V can safely used by FTP transfer. All 201 * others should only be used with great care, so this version will just 202 * ignore the other record formats. F means a fixed number of records per 203 * allocated storage, and V means a variable number of records. 204 * 205 * 206 * Other notes =========== 207 * 208 * The file system supports automatically backup and retrieval of datasets. 209 * If a file is backed up, the ftp LIST command will return: ARCIVE Not 210 * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST 211 * 212 * 213 * Implementation notes ==================== 214 * 215 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning 216 * with F or V, is fully parsed. 217 * 218 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. 219 * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name 220 * FTPFile.Timestamp: change time or null 221 * 222 * 223 * 224 * Additional information ====================== 225 * 226 * The MVS ftp server supports a number of features via the FTP interface. 227 * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2> 228 * SEQ is the default and used for normal file transfer JES is used to 229 * interact with the Job Entry Subsystem (JES) similar to a job scheduler 230 * DB2 is used to interact with a DB2 subsystem 231 * 232 * This parser supports SEQ and JES. 233 * 234 * 235 * 236 * 237 * 238 * 239 */ 240 241 /** 242 * The sole constructor for a MVSFTPEntryParser object. 243 * 244 */ 245 public MVSFTPEntryParser() { 246 super(""); // note the regex is set in preParse. 247 super.configure(null); // configure parser with default configurations 248 } 249 250 /** 251 * Parses a line of an z/OS - MVS FTP server file listing and converts it 252 * into a usable format in the form of an <code> FTPFile </code> instance. 253 * If the file listing line doesn't describe a file, then 254 * <code> null </code> is returned. Otherwise a <code> FTPFile </code> 255 * instance representing the file is returned. 256 * 257 * @param entry 258 * A line of text from the file listing 259 * @return An FTPFile instance corresponding to the supplied entry 260 */ 261 @Override 262 public FTPFile parseFTPEntry(String entry) { 263 boolean isParsed = false; 264 FTPFile f = new FTPFile(); 265 266 if (isType == FILE_LIST_TYPE) { 267 isParsed = parseFileList(f, entry); 268 } else if (isType == MEMBER_LIST_TYPE) { 269 isParsed = parseMemberList(f, entry); 270 if (!isParsed) { 271 isParsed = parseSimpleEntry(f, entry); 272 } 273 } else if (isType == UNIX_LIST_TYPE) { 274 isParsed = parseUnixList(f, entry); 275 } else if (isType == JES_LEVEL_1_LIST_TYPE) { 276 isParsed = parseJeslevel1List(f, entry); 277 } else if (isType == JES_LEVEL_2_LIST_TYPE) { 278 isParsed = parseJeslevel2List(f, entry); 279 } 280 281 if (!isParsed) { 282 f = null; 283 } 284 285 return f; 286 } 287 288 /** 289 * Parse entries representing a dataset list. Only datasets with DSORG PS or 290 * PO or PO-E and with RECFM F* or V* will be parsed. 291 * 292 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred 293 * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 294 * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device 295 * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO 296 * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB 297 * 298 * ----------------------------------- Group within Regex [1] Volume [2] 299 * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record 300 * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: 301 * Dataset organisation. Many exists but only support: PS, PO, PO-E [10] 302 * Dsname: Dataset name 303 * 304 * Note: When volume is ARCIVE, it means the dataset is stored somewhere in 305 * a tape archive. These entries is currently not supported by this parser. 306 * A null value is returned. 307 * 308 * @param file 309 * will be updated with Name, Type, Timestamp if parsed. 310 * @param entry zosDirectoryEntry 311 * @return true: entry was parsed, false: entry was not parsed. 312 */ 313 private boolean parseFileList(FTPFile file, String entry) { 314 if (matches(entry)) { 315 file.setRawListing(entry); 316 String name = group(2); 317 String dsorg = group(1); 318 file.setName(name); 319 320 // DSORG 321 if ("PS".equals(dsorg)) { 322 file.setType(FTPFile.FILE_TYPE); 323 } 324 else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) { 325 // regex already ruled out anything other than PO or PO-E 326 file.setType(FTPFile.DIRECTORY_TYPE); 327 } 328 else { 329 return false; 330 } 331 332 return true; 333 } 334 335 return false; 336 } 337 338 /** 339 * Parse entries within a partitioned dataset. 340 * 341 * Format of a memberlist within a PDS: 342 * <pre> 343 * 0 1 2 3 4 5 6 7 8 344 * Name VV.MM Created Changed Size Init Mod Id 345 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 346 * TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001 347 * 348 * ------------------------------------------- 349 * [1] Name 350 * [2] VV.MM: Version . modification 351 * [3] Created: yyyy / MM / dd 352 * [4,5] Changed: yyyy / MM / dd HH:mm 353 * [6] Size: number of lines 354 * [7] Init: number of lines when first created 355 * [8] Mod: number of modified lines a last save 356 * [9] Id: User id for last update 357 * </pre> 358 * 359 * @param file 360 * will be updated with Name, Type and Timestamp if parsed. 361 * @param entry zosDirectoryEntry 362 * @return true: entry was parsed, false: entry was not parsed. 363 */ 364 private boolean parseMemberList(FTPFile file, String entry) { 365 if (matches(entry)) { 366 file.setRawListing(entry); 367 String name = group(1); 368 String datestr = group(2) + " " + group(3); 369 file.setName(name); 370 file.setType(FTPFile.FILE_TYPE); 371 try { 372 file.setTimestamp(super.parseTimestamp(datestr)); 373 } catch (ParseException e) { 374 e.printStackTrace(); 375 // just ignore parsing errors. 376 // TODO check this is ok 377 return false; // this is a parsing failure too. 378 } 379 return true; 380 } 381 382 return false; 383 } 384 385 /** 386 * Assigns the name to the first word of the entry. Only to be used from a 387 * safe context, for example from a memberlist, where the regex for some 388 * reason fails. Then just assign the name field of FTPFile. 389 * 390 * @param file 391 * @param entry 392 * @return true if the entry string is non-null and non-empty 393 */ 394 private boolean parseSimpleEntry(FTPFile file, String entry) { 395 if (entry != null && entry.trim().length() > 0) { 396 file.setRawListing(entry); 397 String name = entry.split(" ")[0]; 398 file.setName(name); 399 file.setType(FTPFile.FILE_TYPE); 400 return true; 401 } 402 return false; 403 } 404 405 /** 406 * Parse the entry as a standard unix file. Using the UnixFTPEntryParser. 407 * 408 * @param file 409 * @param entry 410 * @return true: entry is parsed, false: entry could not be parsed. 411 */ 412 private boolean parseUnixList(FTPFile file, String entry) { 413 file = unixFTPEntryParser.parseFTPEntry(entry); 414 if (file == null) { 415 return false; 416 } 417 return true; 418 } 419 420 /** 421 * Matches these entries, note: no header: 422 * <pre> 423 * [1] [2] [3] [4] [5] 424 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 425 * 012345678901234567890123456789012345678901234 426 * 1 2 3 4 427 * ------------------------------------------- 428 * Group in regex 429 * [1] Job name 430 * [2] Job number 431 * [3] Job status (INPUT,ACTIVE,OUTPUT) 432 * [4] Number of sysout files 433 * [5] The string "Spool Files" 434 *</pre> 435 * 436 * @param file 437 * will be updated with Name, Type and Timestamp if parsed. 438 * @param entry zosDirectoryEntry 439 * @return true: entry was parsed, false: entry was not parsed. 440 */ 441 private boolean parseJeslevel1List(FTPFile file, String entry) { 442 if (matches(entry)) { 443 if (group(3).equalsIgnoreCase("OUTPUT")) { 444 file.setRawListing(entry); 445 String name = group(2); /* Job Number, used by GET */ 446 file.setName(name); 447 file.setType(FTPFile.FILE_TYPE); 448 return true; 449 } 450 } 451 452 return false; 453 } 454 455 /** 456 * Matches these entries: 457 * <pre> 458 * [1] [2] [3] [4] [5] 459 * JOBNAME JOBID OWNER STATUS CLASS 460 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 461 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 462 * 012345678901234567890123456789012345678901234 463 * 1 2 3 4 464 * ------------------------------------------- 465 * Group in regex 466 * [1] Job name 467 * [2] Job number 468 * [3] Owner 469 * [4] Job status (INPUT,ACTIVE,OUTPUT) 470 * [5] Job Class 471 * [6] The rest 472 * </pre> 473 * 474 * @param file 475 * will be updated with Name, Type and Timestamp if parsed. 476 * @param entry zosDirectoryEntry 477 * @return true: entry was parsed, false: entry was not parsed. 478 */ 479 private boolean parseJeslevel2List(FTPFile file, String entry) { 480 if (matches(entry)) { 481 if (group(4).equalsIgnoreCase("OUTPUT")) { 482 file.setRawListing(entry); 483 String name = group(2); /* Job Number, used by GET */ 484 file.setName(name); 485 file.setType(FTPFile.FILE_TYPE); 486 return true; 487 } 488 } 489 490 return false; 491 } 492 493 /** 494 * preParse is called as part of the interface. Per definition is is called 495 * before the parsing takes place. 496 * Three kind of lists is recognize: 497 * z/OS-MVS File lists 498 * z/OS-MVS Member lists 499 * unix file lists 500 * @since 2.0 501 */ 502 @Override 503 public List<String> preParse(List<String> orig) { 504 // simply remove the header line. Composite logic will take care of the 505 // two different types of 506 // list in short order. 507 if (orig != null && orig.size() > 0) { 508 String header = orig.get(0); 509 if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) { 510 setType(FILE_LIST_TYPE); 511 super.setRegex(FILE_LIST_REGEX); 512 } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) { 513 setType(MEMBER_LIST_TYPE); 514 super.setRegex(MEMBER_LIST_REGEX); 515 } else if (header.indexOf("total") == 0) { 516 setType(UNIX_LIST_TYPE); 517 unixFTPEntryParser = new UnixFTPEntryParser(); 518 } else if (header.indexOf("Spool Files") >= 30) { 519 setType(JES_LEVEL_1_LIST_TYPE); 520 super.setRegex(JES_LEVEL_1_LIST_REGEX); 521 } else if (header.indexOf("JOBNAME") == 0 522 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS 523 setType(JES_LEVEL_2_LIST_TYPE); 524 super.setRegex(JES_LEVEL_2_LIST_REGEX); 525 } else { 526 setType(UNKNOWN_LIST_TYPE); 527 } 528 529 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary 530 orig.remove(0); 531 } 532 } 533 534 return orig; 535 } 536 537 /** 538 * Explicitly set the type of listing being processed. 539 * @param type The listing type. 540 */ 541 void setType(int type) { 542 isType = type; 543 } 544 545 /* 546 * @return 547 */ 548 @Override 549 protected FTPClientConfig getDefaultConfiguration() { 550 return new FTPClientConfig(FTPClientConfig.SYST_MVS, 551 DEFAULT_DATE_FORMAT, null, null, null, null); 552 } 553 554}