1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18 package org.apache.commons.net.ftp.parser;
19
20 import java.text.ParseException;
21 import java.util.List;
22
23 import org.apache.commons.net.ftp.FTPClientConfig;
24 import org.apache.commons.net.ftp.FTPFile;
25
26 /**
27 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS Systems.
28 *
29 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for usage instructions)
30 */
31 public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
32
33 static final int UNKNOWN_LIST_TYPE = -1;
34 static final int FILE_LIST_TYPE = 0;
35 static final int MEMBER_LIST_TYPE = 1;
36 static final int UNIX_LIST_TYPE = 2;
37 static final int JES_LEVEL_1_LIST_TYPE = 3;
38 static final int JES_LEVEL_2_LIST_TYPE = 4;
39
40 /**
41 * Dates are ignored for file lists, but are used for member lists where possible
42 */
43 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
44 // 13:52
45
46 /**
47 * Matches these entries:
48 *
49 * <pre>
50 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
51 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
52 * </pre>
53 *
54 * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a>
55 */
56 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
57 // ignored
58 "\\S+\\s+" + // unit - ignored
59 "\\S+\\s+" + // access date - ignored
60 "\\S+\\s+" + // extents -ignored
61 // If the values are too large, the fields may be merged (NET-639)
62 "(?:\\S+\\s+)?" + // used - ignored
63 "\\S+\\s+" + // recfm - ignored
64 "\\S+\\s+" + // logical record length -ignored
65 "\\S+\\s+" + // block size - ignored
66 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
67 // but only support: PS, PO, PO-E
68 "(\\S+)\\s*"; // Dataset Name (file name)
69
70 /**
71 * Matches these entries:
72 *
73 * <pre>
74 * Name VV.MM Created Changed Size Init Mod Id
75 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001
76 * </pre>
77 */
78 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
79 "\\S+\\s+" + // version, modification (ignored)
80 "\\S+\\s+" + // create date (ignored)
81 "(\\S+)\\s+" + // modification date
82 "(\\S+)\\s+" + // modification time
83 "\\S+\\s+" + // size in lines (ignored)
84 "\\S+\\s+" + // size in lines at creation(ignored)
85 "\\S+\\s+" + // lines modified (ignored)
86 "\\S+\\s*"; // id of user who modified (ignored)
87
88 /**
89 * Matches these entries, note: no header:
90 *
91 * <pre>
92 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
93 * 012345678901234567890123456789012345678901234
94 * 1 2 3 4
95 * </pre>
96 */
97 static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
98 "(\\S+)\\s+" + // job number
99 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
100 "(\\S+)\\s+" + // number of spool files
101 "(\\S+)\\s+" + // Text "Spool" ignored
102 "(\\S+)\\s*" // Text "Files" ignored
103 ;
104
105 /**
106 * JES INTERFACE LEVEL 2 parser Matches these entries:
107 *
108 * <pre>
109 * JOBNAME JOBID OWNER STATUS CLASS
110 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
111 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
112 * </pre>
113 *
114 * Sample output from FTP session:
115 *
116 * <pre>
117 * ftp> quote site filetype=jes
118 * 200 SITE command was accepted
119 * ftp> ls
120 * 200 Port request OK.
121 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
122 * JOBNAME JOBID OWNER STATUS CLASS
123 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
124 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
125 * 250 List completed successfully.
126 * ftp> ls job01906
127 * 200 Port request OK.
128 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
129 * JOBNAME JOBID OWNER STATUS CLASS
130 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000
131 * --------
132 * ID STEPNAME PROCSTEP C DDNAME BYTE-COUNT
133 * 001 JES2 A JESMSGLG 858
134 * 002 JES2 A JESJCL 128
135 * 003 JES2 A JESYSMSG 443
136 * 3 spool files
137 * 250 List completed successfully.
138 * </pre>
139 */
140
141 static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
142 "(\\S+)\\s+" + // job number
143 "(\\S+)\\s+" + // owner ignored
144 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
145 "(\\S+)\\s+" + // job class ignored
146 "(\\S+).*" // rest ignored
147 ;
148
149 private int isType = UNKNOWN_LIST_TYPE;
150
151 /**
152 * Fallback parser for Unix-style listings
153 */
154 private UnixFTPEntryParser unixFTPEntryParser;
155
156 /*
157 * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is
158 * the operating system on the mainframe, and is the new name for MVS)
159 *
160 * The file system on the mainframe does not have hierarchical structure as for example the unix file system. For a more comprehensive description,
161 * please refer to the IBM manuals
162 *
163 * @LINK: http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
164 *
165 *
166 * Dataset names =============
167 *
168 * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be
169 * max 44 characters including the dots.
170 *
171 *
172 * Dataset organisation ====================
173 *
174 * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organisation
175 * (DSORG). There are a number of dataset organisations, but only two are usable for FTP transfer.
176 *
177 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset
178 *
179 * The PS file is just a flat file, as you would find it on the unix file system.
180 *
181 * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It
182 * is possible to CD into the file, and to retrieve the individual members.
183 *
184 *
185 * Dataset record format =====================
186 *
187 * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the
188 * FTP transfer.
189 *
190 * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of
191 * records per allocated storage, and V means a variable number of records.
192 *
193 *
194 * Other notes ===========
195 *
196 * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct
197 * Access Device KJ.IOP998.ERROR.PL.UNITTEST
198 *
199 *
200 * Implementation notes ====================
201 *
202 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed.
203 *
204 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
205 * FTPFile.Timestamp: change time or null
206 *
207 *
208 *
209 * Additional information ======================
210 *
211 * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site
212 * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job
213 * scheduler DB2 is used to interact with a DB2 subsystem
214 *
215 * This parser supports SEQ and JES.
216 */
217
218 /**
219 * The sole constructor for a MVSFTPEntryParser object.
220 */
221 public MVSFTPEntryParser() {
222 super(""); // note the regex is set in preParse.
223 super.configure(null); // configure parser with default configurations
224 }
225
226 /*
227 * @return
228 */
229 @Override
230 protected FTPClientConfig getDefaultConfiguration() {
231 return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null);
232 }
233
234 /**
235 * Parses entries representing a dataset list.
236 * <pre>
237 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10
238 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
239 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
240 * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
241 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU
242 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
243 * Migrated HLQ.DATASET.NAME
244 * </pre>
245 * <pre>
246 * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7]
247 * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organisation. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name
248 * </pre>
249 *
250 * @param entry zosDirectoryEntry
251 * @return null: entry was not parsed.
252 */
253 private FTPFile parseFileList(final String entry) {
254 if (matches(entry)) {
255 final FTPFile file = new FTPFile();
256 file.setRawListing(entry);
257 final String name = group(2);
258 final String dsorg = group(1);
259 file.setName(name);
260
261 // DSORG
262 if ("PS".equals(dsorg)) {
263 file.setType(FTPFile.FILE_TYPE);
264 } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
265 // regex already ruled out anything other than PO or PO-E
266 file.setType(FTPFile.DIRECTORY_TYPE);
267 } else {
268 return null;
269 }
270
271 return file;
272 }
273
274 final boolean migrated = entry.startsWith("Migrated");
275 if (migrated || entry.startsWith("ARCIVE")) {
276 // Type of file is unknown for migrated datasets
277 final FTPFile file = new FTPFile();
278 file.setRawListing(entry);
279 file.setType(FTPFile.UNKNOWN_TYPE);
280 file.setName(entry.split("\\s+")[migrated ? 1 : 5]);
281 return file;
282 }
283
284 return null;
285 }
286
287 /**
288 * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an <code> FTPFile </code> instance. If the
289 * file listing line doesn't describe a file, then <code> null </code> is returned. Otherwise, a <code> FTPFile </code> instance representing the file is
290 * returned.
291 *
292 * @param entry A line of text from the file listing
293 * @return An FTPFile instance corresponding to the supplied entry
294 */
295 @Override
296 public FTPFile parseFTPEntry(final String entry) {
297 switch (isType) {
298 case FILE_LIST_TYPE:
299 return parseFileList(entry);
300 case MEMBER_LIST_TYPE:
301 return parseMemberList(entry);
302 case UNIX_LIST_TYPE:
303 return unixFTPEntryParser.parseFTPEntry(entry);
304 case JES_LEVEL_1_LIST_TYPE:
305 return parseJeslevel1List(entry);
306 case JES_LEVEL_2_LIST_TYPE:
307 return parseJeslevel2List(entry);
308 default:
309 break;
310 }
311
312 return null;
313 }
314
315 /**
316 * Matches these entries, note: no header:
317 *
318 * <pre>
319 * [1] [2] [3] [4] [5]
320 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
321 * 012345678901234567890123456789012345678901234
322 * 1 2 3 4
323 * -------------------------------------------
324 * Group in regex
325 * [1] Job name
326 * [2] Job number
327 * [3] Job status (INPUT,ACTIVE,OUTPUT)
328 * [4] Number of sysout files
329 * [5] The string "Spool Files"
330 * </pre>
331 *
332 * @param entry zosDirectoryEntry
333 * @return null: entry was not parsed.
334 */
335 private FTPFile parseJeslevel1List(final String entry) {
336 return parseJeslevelList(entry, 3);
337 }
338
339 /**
340 * Matches these entries:
341 *
342 * <pre>
343 * [1] [2] [3] [4] [5]
344 * JOBNAME JOBID OWNER STATUS CLASS
345 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
346 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
347 * 012345678901234567890123456789012345678901234
348 * 1 2 3 4
349 * -------------------------------------------
350 * Group in regex
351 * [1] Job name
352 * [2] Job number
353 * [3] Owner
354 * [4] Job status (INPUT,ACTIVE,OUTPUT)
355 * [5] Job Class
356 * [6] The rest
357 * </pre>
358 *
359 * @param entry zosDirectoryEntry
360 * @return null: entry was not parsed.
361 */
362 private FTPFile parseJeslevel2List(final String entry) {
363 return parseJeslevelList(entry, 4);
364 }
365
366 private FTPFile parseJeslevelList(final String entry, final int matchNum) {
367 if (matches(entry)) {
368 final FTPFile file = new FTPFile();
369 if (group(matchNum).equalsIgnoreCase("OUTPUT")) {
370 file.setRawListing(entry);
371 final String name = group(2); /* Job Number, used by GET */
372 file.setName(name);
373 file.setType(FTPFile.FILE_TYPE);
374 return file;
375 }
376 }
377 return null;
378 }
379
380 /**
381 * Parses entries within a partitioned dataset.
382 *
383 * Format of a memberlist within a PDS:
384 *
385 * <pre>
386 * 0 1 2 3 4 5 6 7 8
387 * Name VV.MM Created Changed Size Init Mod Id
388 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001
389 * TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001
390 *
391 * -------------------------------------------
392 * [1] Name
393 * [2] VV.MM: Version . modification
394 * [3] Created: yyyy / MM / dd
395 * [4,5] Changed: yyyy / MM / dd HH:mm
396 * [6] Size: number of lines
397 * [7] Init: number of lines when first created
398 * [8] Mod: number of modified lines a last save
399 * [9] Id: User id for last update
400 * </pre>
401 *
402 * @param entry zosDirectoryEntry
403 * @return null: entry was not parsed.
404 */
405 private FTPFile parseMemberList(final String entry) {
406 final FTPFile file = new FTPFile();
407 if (matches(entry)) {
408 file.setRawListing(entry);
409 final String name = group(1);
410 final String datestr = group(2) + " " + group(3);
411 file.setName(name);
412 file.setType(FTPFile.FILE_TYPE);
413 try {
414 file.setTimestamp(super.parseTimestamp(datestr));
415 } catch (final ParseException e) {
416 // just ignore parsing errors.
417 // TODO check this is ok
418 // Drop thru to try simple parser
419 }
420 return file;
421 }
422
423 /*
424 * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason
425 * fails. Then just assign the name field of FTPFile.
426 */
427 if (entry != null && !entry.trim().isEmpty()) {
428 file.setRawListing(entry);
429 final String name = entry.split(" ")[0];
430 file.setName(name);
431 file.setType(FTPFile.FILE_TYPE);
432 return file;
433 }
434 return null;
435 }
436
437 /**
438 * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized:
439 * <ul>
440 * <li>z/OS-MVS File lists,</li>
441 * <li>z/OS-MVS Member lists,</li>
442 * <li>unix file lists.</li>
443 * </ul>
444 * @since 2.0
445 */
446 @Override
447 public List<String> preParse(final List<String> orig) {
448 // simply remove the header line. Composite logic will take care of the
449 // two different types of
450 // list in short order.
451 if (orig != null && !orig.isEmpty()) {
452 final String header = orig.get(0);
453 if (header.contains("Volume") && header.contains("Dsname")) {
454 setType(FILE_LIST_TYPE);
455 super.setRegex(FILE_LIST_REGEX);
456 } else if (header.contains("Name") && header.contains("Id")) {
457 setType(MEMBER_LIST_TYPE);
458 super.setRegex(MEMBER_LIST_REGEX);
459 } else if (header.indexOf("total") == 0) {
460 setType(UNIX_LIST_TYPE);
461 unixFTPEntryParser = new UnixFTPEntryParser();
462 } else if (header.indexOf("Spool Files") >= 30) {
463 setType(JES_LEVEL_1_LIST_TYPE);
464 super.setRegex(JES_LEVEL_1_LIST_REGEX);
465 } else if (header.indexOf("JOBNAME") == 0 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
466 setType(JES_LEVEL_2_LIST_TYPE);
467 super.setRegex(JES_LEVEL_2_LIST_REGEX);
468 } else {
469 setType(UNKNOWN_LIST_TYPE);
470 }
471
472 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
473 orig.remove(0);
474 }
475 }
476
477 return orig;
478 }
479
480 /**
481 * Sets the type of listing being processed.
482 *
483 * @param type The listing type.
484 */
485 void setType(final int type) {
486 isType = type;
487 }
488
489 }