View Javadoc

1   //Copyright 2006-2007, by the California Institute of Technology.
2   //ALL RIGHTS RESERVED. United States Government Sponsorship acknowledged.
3   //Any commercial use must be negotiated with the Office of Technology Transfer
4   //at the California Institute of Technology.
5   //
6   //This software is subject to U. S. export control laws and regulations
7   //(22 C.F.R. 120-130 and 15 C.F.R. 730-774). To the extent that the software
8   //is subject to U.S. export control laws and regulations, the recipient has
9   //the responsibility to obtain export licenses or other export authority as
10  //may be required before exporting such information to foreign countries or
11  //providing access to foreign nationals.
12  
13  // $Id: VTool.java 2934 2007-09-27 20:08:54Z mcayanan $
14  //
15  
16  package gov.nasa.pds.vtool;
17  
18  
19  import gov.nasa.pds.tools.dict.Dictionary;
20  import gov.nasa.pds.tools.dict.parser.DictionaryParser;
21  import gov.nasa.pds.tools.file.FileList;
22  import gov.nasa.pds.tools.file.FileListGenerator;
23  import gov.nasa.pds.tools.label.Label;
24  import gov.nasa.pds.tools.label.parser.LabelParser;
25  import gov.nasa.pds.tools.label.parser.LabelParserFactory;
26  import gov.nasa.pds.tools.label.validate.Status;
27  import gov.nasa.pds.tools.license.ToolsLicense;
28  import gov.nasa.pds.tools.logging.ToolsLevel;
29  import gov.nasa.pds.tools.logging.ToolsLogFormatter;
30  import gov.nasa.pds.tools.logging.ToolsLogRecord;
31  import gov.nasa.pds.tools.options.ToolsOption;
32  import gov.nasa.pds.tools.report.Report;
33  import gov.nasa.pds.tools.report.StyleSheet;
34  import gov.nasa.pds.tools.time.ToolsTime;
35  import gov.nasa.pds.tools.util.Utility;
36  import gov.nasa.pds.vtool.config.VToolConfigKeys;
37  import gov.nasa.pds.vtool.flags.VToolFlags;
38  import gov.nasa.pds.vtool.status.ExitStatus;
39  import gov.nasa.pds.vtool.status.ExitStatusType;
40  import gov.nasa.pds.vtool.validate.UnknownLabelStatusException;
41  import gov.nasa.pds.vtool.validate.ValidationRecord;
42  import gov.nasa.pds.vtool.validate.Validator;
43  import gov.nasa.pds.vtool.validate.ValidatorFactory;
44  import java.io.ByteArrayInputStream;
45  import java.io.ByteArrayOutputStream;
46  import java.io.File;
47  import java.io.FileInputStream;
48  import java.io.FileNotFoundException;
49  import java.io.FileOutputStream;
50  import java.io.IOException;
51  import java.io.InputStream;
52  import java.io.OutputStream;
53  import java.net.MalformedURLException;
54  import java.net.URL;
55  import java.text.SimpleDateFormat;
56  import java.util.ArrayList;
57  import java.util.Arrays;
58  import java.util.Iterator;
59  import java.util.List;
60  import java.util.logging.FileHandler;
61  import java.util.logging.Handler;
62  import java.util.logging.Level;
63  import java.util.logging.Logger;
64  import java.util.logging.StreamHandler;
65  
66  import javax.xml.transform.TransformerException;
67  
68  import org.apache.commons.cli.CommandLine;
69  import org.apache.commons.cli.CommandLineParser;
70  import org.apache.commons.cli.GnuParser;
71  import org.apache.commons.cli.HelpFormatter;
72  import org.apache.commons.cli.Options;
73  import org.apache.commons.cli.ParseException;
74  import org.apache.commons.configuration.AbstractConfiguration;
75  import org.apache.commons.configuration.Configuration;
76  import org.apache.commons.configuration.ConfigurationException;
77  import org.apache.commons.configuration.ConversionException;
78  import org.apache.commons.configuration.PropertiesConfiguration;
79  
80  /***
81   * Class to perform automated validation to determine if a given data product
82   * is PDS compliant.
83   * <p>
84   * This replaces LVTool functionality.
85   *  
86   * @author mcayanan
87   * @version $Revision: 2934 $
88   *
89   *
90   */
91  public class VTool implements VToolConfigKeys, VToolFlags, Status, 
92                                         ExitStatusType, ToolsLicense, StyleSheet {
93  	private final String VERSION_ID = "1.1.0";
94  	private final String FILE_REP = "*";
95  	private static Logger log = Logger.getLogger(VTool.class.getName());
96  	private ByteArrayOutputStream byteStream;
97  	private String currDir;
98  	private List logHandlers;
99  	private Options options;
100 	//TODO: Flags to be implemented: data object(-O,--no-obj),
101 	private boolean alias;
102 	private URL config;
103 //	private boolean dataObj;
104 	private List dictionaries;
105 	private List userSpecifiedDictionaries;
106 	private List noFiles;
107 	private List noDirs;
108 	private boolean followPtrs;
109 	private List includePaths;
110 	private boolean force;
111 	private boolean progress;
112 	private List regexp;
113 	private boolean recursive;
114 	private List targets;
115 	private File logFile;
116 	private boolean showLog;
117 	private File rptFile;
118 	private String rptStyle;
119 	private short verbose;
120 	private String severity;
121 	private boolean dictionaryPassed;
122 
123 
124 	/*** 
125 	 * Default constructor.
126 	 */
127 	public VTool() {
128 		alias = false;
129 		config = null;
130 		dictionaries = new ArrayList();
131 		userSpecifiedDictionaries = null;
132 //		dataObj = true;
133 		noFiles = null;
134 		noDirs = null;
135 		followPtrs = true;
136 		targets = null;
137 		includePaths = null;
138 		force = false;
139 		progress = false;
140 		regexp = null;
141 		recursive = true;
142 		rptFile = null;
143 		rptStyle = "full";
144 		severity = "Warning";
145 		logFile = null;
146 		showLog = false;
147 		verbose = 2;
148 		currDir = null;
149 		buildOpts();
150 		logHandlers = new ArrayList();
151 		dictionaryPassed = true;
152 	}
153 	
154 	/***
155 	 * Show the version and disclaimer notice for VTool. 
156 	 *
157 	 */	
158 	public void showVersion() {
159 		System.out.println("\nPDS Validation Tool (VTool) " + VERSION_ID);
160 		System.out.println(LICENSE);
161 	}
162 	
163 	/***
164 	 * Display VTool usage and help information
165 	 *
166 	 */
167 	public void showHelp() {
168 		HelpFormatter formatter = new HelpFormatter();
169 		formatter.printHelp(37, "VTool", null, options, null);
170 	}
171 	
172 	/***
173 	 * Builds the set of configurable parameters for VTool.
174 	 */
175 	private void buildOpts() {
176 		options = new Options();
177 		ToolsOption opt = null;
178 		char separator = ' ';
179 		
180 		options.addOption(new ToolsOption(ALIAS[SHORT], ALIAS[LONG], WHATIS_ALIAS));
181 		options.addOption(new ToolsOption(FOLLOW[SHORT], FOLLOW[LONG],WHATIS_FOLLOW));
182 		options.addOption(new ToolsOption(HELP[SHORT], HELP[LONG], WHATIS_HELP));
183 		options.addOption(new ToolsOption(PARTIAL[SHORT], PARTIAL[LONG], WHATIS_PARTIAL)); 
184 		options.addOption(new ToolsOption(LOCAL[SHORT], LOCAL[LONG], WHATIS_LOCAL)); 
185 		options.addOption(new ToolsOption(PROGRESS[SHORT], PROGRESS[LONG], WHATIS_PROGRESS)); 
186 		options.addOption(new ToolsOption(VERSION[SHORT], VERSION[LONG], WHATIS_VERSION)); 	
187 		
188 		// These are options that require an argument
189 		
190 		// Option to specify a configuration file
191 		opt = new ToolsOption(CONFIG[SHORT], CONFIG[LONG], WHATIS_CONFIG);
192 		opt.hasArg(CONFIG[ARGNAME], String.class);
193 		options.addOption(opt);
194 		
195 		// Option to specify the PSDD and any local dictionaries
196 		opt = new ToolsOption(DICT[SHORT], DICT[LONG], WHATIS_DICT);
197 		opt.hasArgs(DICT[ARGNAME], String.class, separator);
198 		options.addOption(opt);
199 		
200 		// Option to specify a file containing a list of file extensions to ignore
201 		opt = new ToolsOption(IGNOREFILE[SHORT], IGNOREFILE[LONG],WHATIS_IGNOREFILE);
202 		opt.hasArgs(IGNOREFILE[ARGNAME], String.class, separator);
203 		options.addOption(opt);
204 		
205 		// Option to specify the label(s) to validate
206 		opt = new ToolsOption(TARGET[SHORT], TARGET[LONG], WHATIS_TARGET);
207 		opt.hasArgs(TARGET[ARGNAME], String.class, separator);
208 		options.addOption(opt);
209 		
210 		// Option to specify a pattern to match against the input directory to be validated
211 		opt = new ToolsOption(REGEXP[SHORT], REGEXP[LONG], WHATIS_REGEXP);
212 		opt.hasArgs(REGEXP[ARGNAME], String.class, separator);
213 		options.addOption(opt);		
214 		
215 		// Option to specify a path to the Pointer files	
216 		opt = new ToolsOption(INCLUDES[SHORT], INCLUDES[LONG], WHATIS_INCLUDES);
217 		opt.hasArgs(INCLUDES[ARGNAME], String.class, separator);
218 		options.addOption(opt);
219 
220 		//Option to specify a file containing a list of directories to ignore
221 		opt = new ToolsOption(IGNOREDIR[SHORT], IGNOREDIR[LONG], WHATIS_IGNOREDIR);
222 		opt.hasArgs(IGNOREDIR[ARGNAME], String.class, separator);
223 		options.addOption(opt);
224 		
225 		// Option to specify the log file name
226 		opt = new ToolsOption(LOG[SHORT], LOG[LONG], WHATIS_LOG);
227 		opt.hasArg(LOG[ARGNAME], String.class, true);
228 		options.addOption(opt);
229 		
230 		//Option to specify the report file name
231 		opt = new ToolsOption(REPORT[SHORT], REPORT[LONG], WHATIS_REPORT);
232 		opt.hasArg(REPORT[ARGNAME], String.class);
233 		options.addOption(opt);
234 				
235 		// Option to specify how detail the reporting should be
236 		opt = new ToolsOption(RPTSTYLE[SHORT], RPTSTYLE[LONG], WHATIS_RPTSTYLE);
237 		opt.hasArg(RPTSTYLE[ARGNAME], String.class);
238 		options.addOption(opt);
239 		
240 		// Option to specify the severity level and above
241 		opt = new ToolsOption(VERBOSE[SHORT], VERBOSE[LONG], WHATIS_VERBOSE);
242 		opt.hasArg(VERBOSE[ARGNAME], short.class);
243 		options.addOption(opt);	
244 	}
245 	
246 	/***
247 	 * Parses the VTool command-line.
248 	 * @param argv arguments given on the command-line
249 	 * @throws ApplicationException 
250 	 */
251 	public CommandLine parseLine(String[] argv) throws ApplicationException {
252 		CommandLineParser parser = new GnuParser();
253 		try {
254 			return parser.parse(options, argv);
255 		}
256 		catch( ParseException exp ) {
257 			throw new ApplicationException("Command line parser failed.\n\nReason: " + exp.getMessage() );
258 		}
259 	}
260 	
261 	/*** 
262 	 * Queries the VTool command-line.
263 	 * @throws ApplicationException 
264 	 * @throws MalformedURLException 
265 	 *
266 	 */
267 	public void queryCmdLine(CommandLine cmd) throws ApplicationException, 
268 	                                                 MalformedURLException {
269 		List targetList = new ArrayList();
270 		
271 		// Check if the help flag was set
272 		if (cmd.hasOption(HELP[SHORT])) {
273 			showHelp();
274 			System.exit(SUCCESS);
275 		} else if(cmd.hasOption(VERSION[SHORT])) {
276 		// Check if the flag to display the version number and disclaimer
277 		// notice was set
278 			showVersion();
279 			System.exit(SUCCESS);
280 		}
281 		else {
282 			// Check if a configuration file was specified
283 			if (cmd.hasOption(CONFIG[SHORT])) {
284 				URL file = Utility.toURL(cmd.getOptionValue(CONFIG[SHORT]));
285 				readConfigFile(file);
286 			}
287 			// Check for the 'l' flag to specify a file where the log will be
288 			// written to
289 			if (cmd.hasOption(LOG[SHORT])) {
290 				if (cmd.getOptionValue(LOG[SHORT]) != null)
291 					setLogFile(new File(cmd.getOptionValue(LOG[SHORT])));
292 				else
293 					setShowLog(true);
294 			}
295 			// verbose flag
296 			if (cmd.hasOption(VERBOSE[SHORT])) {
297 				try {
298 					setVerbose(Short.parseShort(cmd.getOptionValue(VERBOSE[SHORT])));
299 				}
300 				catch(NumberFormatException ne) {
301 					throw new ApplicationException("Problems parsing value for the 'v' flag: " 
302 							+ cmd.getOptionValue(VERBOSE[SHORT]));
303 				}
304 				catch(IllegalArgumentException ae) {
305 					throw new ApplicationException(ae.getMessage());
306 				}
307 		
308 			}
309 			
310 			//Grab the targets specified implicitly
311 			if (cmd.getArgList().size() != 0)
312 				targetList.addAll(cmd.getArgList());
313 			//Grab the targets specified explicitly
314 			if (cmd.hasOption(TARGET[SHORT])) {
315 				targetList.addAll(
316 						Arrays.asList(cmd.getOptionValues(TARGET[SHORT])));
317 			}
318 			setTargets(targetList);
319 			
320 			// Check to see if aliasing is turned on
321 			if (cmd.hasOption(ALIAS[SHORT]))
322 				setAlias(true);
323 			// Check for the flag that indicates whether to follow pointers
324 			if (cmd.hasOption(FOLLOW[SHORT]))
325 				setFollowPtrs(false);
326 			// Check for the include paths to indicate the paths to search for 
327 			// when following pointers
328 			if (cmd.hasOption(INCLUDES[SHORT]))
329 				setIncludePaths(
330 						Arrays.asList(cmd.getOptionValues(INCLUDES[SHORT])));
331 			// TODO: Check later when data object validation is implemented
332 			// Check to see if data object validation is set
333 //			if (cmd.hasOption("O")) 
334 //				setDataObj(false);
335 			if (cmd.hasOption(PROGRESS[SHORT]))
336 				setProgress(true);
337 			// Check to see if VTool will not recurse down a directory tree
338 			if (cmd.hasOption(LOCAL[SHORT]))
339 				setRecursive(false);
340 			// Check to see if regular expressions were set
341 			if (cmd.hasOption(REGEXP[SHORT]))
342 				setRegexp(Arrays.asList(cmd.getOptionValues(REGEXP[SHORT])));
343 			// Check to get file that contains file patterns to ignore
344 			if (cmd.hasOption(IGNOREFILE[SHORT])) {
345 				setNoFiles(Arrays.asList(
346 						cmd.getOptionValues(IGNOREFILE[SHORT])));
347 			}
348 			// Check to get file that contains directory patterns to ignore
349 			if (cmd.hasOption(IGNOREDIR[SHORT])) {
350 				setNoDirs(Arrays.asList(cmd.getOptionValues(IGNOREDIR[SHORT])));
351 			}
352 			// Check to get the dictionary file(s) 
353 			if (cmd.hasOption(DICT[SHORT]))
354 				setDictionaries(Arrays.asList(cmd.getOptionValues(DICT[SHORT])));
355 			// Check to see what type of reporting style the report will have
356 			if (cmd.hasOption(RPTSTYLE[SHORT]))				
357 				setRptStyle(cmd.getOptionValue(RPTSTYLE[SHORT]));
358 			// Check to see where the human-readable report will go
359 			if (cmd.hasOption(REPORT[SHORT]))
360 				setRptFile(new File(cmd.getOptionValue(REPORT[SHORT])));
361 			// Check to see if standalone label fragment validation is enabled
362 			if (cmd.hasOption(PARTIAL[SHORT]))
363 				setForcePartial(true);
364 		}
365 	}
366 	
367 	/***
368 	 * Set aliasing flag.
369 	 * @param a 'false' if aliasing should be turned off, 
370 	 * 'true' otherwise
371 	 */
372 	public void setAlias(boolean a) {
373 		this.alias = a;
374 	}
375 	
376 /*
377 	public void setDataObj(boolean d) {
378 		this.dataObj = d;
379 	}
380 */
381 
382 	/***
383 	 * Set the dictionary file names passed into VTool .
384 	 * @param d a List object of dictionary files
385 	 * @throws ApplicationException 
386 	 */
387 	public void setDictionaries(List d) throws ApplicationException {
388 		URL url = null;
389 		for(Iterator i=d.iterator(); i.hasNext();) {
390 			try {
391 				url = Utility.toURL(i.next().toString());
392 				url.openStream().close();
393 			}catch(IOException eX) {
394 				throw new ApplicationException(eX.getMessage());
395 			}
396 			this.dictionaries.add(url);
397 		}
398 		//This is more for a user's information. This will store
399 		//what the user originally specified on the command-line.
400 		userSpecifiedDictionaries = d;
401 	}
402 	
403 	/***
404 	 * Set the flag that determines whether to follow pointers found in
405 	 * a label.
406 	 * @param f 'true' to follow, 'false' otherwise
407 	 */
408 	public void setFollowPtrs(boolean f) {
409 		this.followPtrs = f;
410 	}
411 	
412 	/***
413 	 * Set the flag that determines whether to validate standalone label.
414 	 * fragments
415 	 * @param f 'true' to enable, 'false' otherwise
416 	 */
417 	public void setForcePartial(boolean f) {
418 		this.force = f;
419 	}
420 	
421 	/***
422 	 * Set the paths to search for files referenced by pointers. 
423 	 * <p>
424 	 * Default is to always look first in the same directory
425 	 * as the label, then search specified directories.
426 	 * @param i List of paths
427 	 */
428 	public void setIncludePaths(List i) {
429 		this.includePaths = i;
430 	}
431 	
432 	/***
433 	 * Set the flag to ignore specified directories.
434 	 * @param f a text file containing a list of directories and/or directory
435 	 *  patterns to ignore during validation. The names must be listed one
436 	 *  name per line.
437 	 */
438 	public void setNoDirs(List f) {
439 		this.noDirs = f;
440 	}
441 	
442 	/***
443 	 * Set the flag to ignore specified files.
444 	 * @param f a list of files/file patterns to ignore during validation.
445 	 */
446 	public void setNoFiles(List f) {
447 		this.noFiles = f;
448 	}
449 	
450 	/***
451 	 * Set the file name for the machine-readable log.
452 	 * @param f file name of the log
453 	 */
454 	public void setLogFile(File f) {
455 		this.logFile = f;
456 	}
457 	
458 	/***
459 	 * Set the flag to write the log to standard out.
460 	 * @param l 
461 	 */
462 	public void setShowLog(boolean l) {
463 		this.showLog = l;
464 	}
465 	
466 	/***
467 	 * Set the file for the human-readable report.
468 	 * @param f file name
469 	 */
470 	public void setRptFile(File f) {
471 		this.rptFile = f;
472 	}
473 	
474 	/***
475 	 * Set the output style for the report.
476 	 * @param style 'sum' for a summary report, 'min' for a minimal report,
477 	 *  and 'full' for a full report
478 	 * @throws ApplicationException 
479 	 */
480 	public void setRptStyle(String style) throws ApplicationException {
481 		if ( (style.equalsIgnoreCase("sum") == false) &&
482 				(style.equalsIgnoreCase("min") == false) &&
483 				(style.equalsIgnoreCase("full") == false) ) {
484 				throw new ApplicationException(
485 						"Invalid value entered for 's' flag. Value can only "
486 						+ "be either 'full', 'sum', or 'min'");
487 		}
488 		this.rptStyle = style;
489 	}
490 	
491 	/***
492 	 * Set the progress reporting flag.
493 	 * @param p
494 	 */
495 	public void setProgress(boolean p) {
496 		this.progress = p;
497 	}
498 	
499 	/***
500 	 * Set the patterns flag.
501 	 * @param e a List of patterns to be matched when searching for files to
502 	 *  validate in a directory
503 	 */
504 	public void setRegexp(List e) {
505 		this.regexp = e;
506 	}
507 	
508 	/***
509 	 * Set the recursive flag.
510 	 * @param r 'true' to recursively traverse down a directory and all its
511 	 *  sub-directories, 'false' otherwise
512 	 */
513 	public void setRecursive(boolean r) {
514 		this.recursive = r;
515 	}
516 	
517 	/***
518 	 * Set the targets flag.
519 	 * @param t a List of files, URLs, and/or directories to be validated
520 	 */
521 	public void setTargets(List t) {
522 		this.targets = t;
523 	}
524 	
525 	/***
526 	 * Set the verbosity level and above to include in the reporting.
527 	 * @param v '1' for info, '2' for warnings, and '3' for errors
528 	 * @throws ApplicationException 
529 	 */
530 	public void setVerbose(short v) throws ApplicationException {
531 		if (v < 1 || v > 3) {
532 			throw new ApplicationException(
533 					"Invalid value entered for 'v' flag. "
534 					+ "Valid values can only be 1, 2, or 3");
535 		}
536 		verbose = v;
537 
538 		if (verbose == 1)
539 			severity = new String("Info");
540 		else if (verbose == 2)
541 			severity = new String("Warning");
542 		else if (verbose == 3)
543 			severity = new String("Error");
544 	}
545 	
546 	/***
547 	 * Reads a configuration file to set the default behaviors for VTool.
548 	 * <p>
549 	 * Flags set on the command-line will override flags set in the 
550 	 * configuration file
551 	 * 
552 	 * @param file a file containing keyword/value statements
553 	 * @throws ApplicationException 
554 	 */
555 	public void readConfigFile(URL file) throws ApplicationException {
556 		Configuration config = null;
557 		AbstractConfiguration.setDelimiter(' ');
558 		
559 		try {
560 			config = new PropertiesConfiguration(file);
561 		} catch(ConfigurationException eX) {
562 			throw new ApplicationException(eX.getMessage());
563 		}
564 		
565 		try {
566 			if (config.isEmpty())
567 				throw new ApplicationException("Configuration file is empty: "
568 						+ file.toString());
569 			if (config.containsKey(ALIASKEY))
570 				setAlias(config.getBoolean(ALIASKEY));
571 			//TODO: Get Data Object key value when data object validation is implemented
572 //			if (config.containsKey(DATAOBJKEY))
573 //				setDataObj(config.getBoolean(DATAOBJKEY));
574 			if (config.containsKey(DICTKEY))
575 				setDictionaries(config.getList(DICTKEY));
576 			if (config.containsKey(FORCEKEY))
577 				setForcePartial(config.getBoolean(FORCEKEY));
578 			if (config.containsKey(FOLLOWKEY))
579 				setFollowPtrs(config.getBoolean(FOLLOWKEY));
580 			if (config.containsKey(IGNOREDIRKEY)) {
581 				setNoDirs(config.getList(IGNOREDIRKEY));
582 				// Removes quotes surrounding each pattern being specified
583 				for(int i=0; i < this.noDirs.size(); i++) {
584 					this.noDirs.set(i, 
585 							this.noDirs.get(i).toString().replace('"', ' ').trim());
586 				}
587 			}
588 			if (config.containsKey(IGNOREFILEKEY)) {
589 				setNoFiles(config.getList(IGNOREFILEKEY));
590 				// Removes quotes surrounding each pattern being specified
591 				for(int i=0; i < noFiles.size(); i++) {
592 					this.noFiles.set(i, 
593 							this.noFiles.get(i).toString().replace('"', ' ').trim());
594 				}
595 			}
596 			if (config.containsKey(INCLUDESKEY))
597 				setIncludePaths(config.getList(INCLUDESKEY));
598 			if (config.containsKey(LOGKEY)) {
599 				String logValue = new String(config.getProperty(LOGKEY).toString());
600 				// Check for a boolean value or a file name
601 				if(logValue.equalsIgnoreCase("true") 
602 						|| logValue.equalsIgnoreCase("false")) {
603 					setShowLog(Boolean.valueOf(logValue).booleanValue());
604 				}
605 				else
606 					setLogFile(new File(logValue));
607 			}
608 			if (config.containsKey(PROGRESSKEY))
609 				setProgress(config.getBoolean(PROGRESSKEY));
610 			if (config.containsKey(RECURSEKEY))
611 				setRecursive(config.getBoolean(RECURSEKEY));
612 			if (config.containsKey(REGEXPKEY)) {
613 				setRegexp(config.getList(REGEXPKEY));
614 				// Removes quotes surrounding each pattern being specified
615 				for(int i=0; i < this.regexp.size(); i++) {
616 					this.regexp.set(i, 
617 							this.regexp.get(i).toString().replace('"', ' ').trim());
618 				}
619 			}
620 			if (config.containsKey(REPORTKEY))
621 				setRptFile(new File(config.getString(REPORTKEY)));
622 			if (config.containsKey(STYLEKEY))
623 				setRptStyle(config.getString(STYLEKEY));
624 			if (config.containsKey(TARGETKEY))
625 				setTargets(config.getList(TARGETKEY));
626 			if (config.containsKey(VERBOSEKEY))
627 				setVerbose(config.getShort(VERBOSEKEY));
628 		} catch(ConversionException eX) {
629 			throw new ApplicationException(eX.getMessage());
630 		}
631 	}
632 	
633 	/***
634 	 * Routine to store a message into the logger.
635 	 * @param level The severity level of the message.
636 	 * @param msg The message to log.
637 	 */
638 	public void logMessage(Level level, String msg) {
639 		logMessage(level, msg, null);
640 	}
641 	
642 	/***
643 	 * Routine to store a message into the logger.
644 	 * @param level The severity level of the message.
645 	 * @param msg The message to log.
646 	 * @param file The file name associated with the message being logged.
647 	 */
648 	public void logMessage(Level level, String msg, String file) {
649 		log.log(new ToolsLogRecord(level, msg, file));
650 	}
651 	
652 	/***
653 	 * Logs report header information such as version of the tool, 
654 	 * execution time, and flag settings.
655 	 * @throws SystemException 
656 	 *
657 	 */
658 	public void logRptHeader() throws SystemException {
659 		ToolsTime time = new ToolsTime();
660 
661 		//TODO: Print out data object validation
662 		logMessage(Level.CONFIG, "VTool Version             " + VERSION_ID);
663 		try {
664 			logMessage(Level.CONFIG, 
665 				"Execution Date            " 
666 				+ time.getTime(new SimpleDateFormat("EEE, MMM dd yyyy 'at' HH:mm:ss a")));
667 		} catch(IllegalArgumentException eX) {
668 			throw new SystemException(eX.getMessage());
669 		}		
670 		logMessage(ToolsLevel.PARAMETER, 
671 				"Target(s)                      " + targets);
672 		if (userSpecifiedDictionaries != null) {
673 			logMessage(ToolsLevel.PARAMETER, 
674 					"Dictionary File(s)             " + userSpecifiedDictionaries);
675 		}
676 		logMessage(ToolsLevel.PARAMETER, 
677 				"Aliasing                       " + alias);
678 		logMessage(ToolsLevel.PARAMETER, 
679 				"Directory Recursion            " + recursive);
680 		logMessage(ToolsLevel.PARAMETER, 
681 				"Follow Pointers                " + followPtrs);
682 		logMessage(ToolsLevel.PARAMETER, 
683 				"Validate Standalone Fragments  " + force);
684 		
685 		if (logFile != null) {
686 			logMessage(ToolsLevel.PARAMETER, 
687 					"Log File                       " + logFile);
688 		}
689 		if (rptFile != null) {
690 			logMessage(ToolsLevel.PARAMETER, 
691 					"Report File                    " + rptFile);
692 		}
693 		if (rptStyle != null) {
694 		    logMessage(ToolsLevel.PARAMETER, 
695 		    		"Report Style                   " + rptStyle);
696 		}
697 		if (severity != null) {
698 			logMessage(ToolsLevel.PARAMETER, 
699 					"Severity Level                 " + severity);
700 		}
701 		if (includePaths != null) {
702 			logMessage(ToolsLevel.PARAMETER, 
703 					"Include Path(s)                " + includePaths);
704 		}
705 		if (config != null) {
706 			logMessage(ToolsLevel.PARAMETER, 
707 					"Configuration File             " + config);
708 		}
709 		if (regexp != null) {
710 			logMessage(ToolsLevel.PARAMETER, 
711 					"Files Patterns                 " + regexp);
712 		}
713 		if (noDirs != null) {
714 			logMessage(ToolsLevel.PARAMETER, 
715 					"Excluded Directories           " + noDirs);
716 		}
717 		if (noFiles != null) {
718 			logMessage(ToolsLevel.PARAMETER, 
719 					"Excluded Files                 " + noFiles);
720 		}
721 		logMessage(ToolsLevel.PARAMETER, 
722 				"Progress Reporting             " + progress);
723 	}
724 	
725 	/***
726 	 * Configures the logger appropriately.
727 	 * <p>
728 	 * If a log file was specified on the command-line, the log
729 	 * will be written to that file. If the log flag was
730 	 * specified with no file spec, then the log will be written
731 	 * to standard out. Otherwise, the log will be written to
732 	 * memory (ByteArrayOutputStream). 
733 	 * @throws SystemException 
734 	 * @throws ApplicationException 
735 	 *
736 	 */
737 	public void setupLogger() 
738 	           throws ApplicationException, SystemException {
739 		Logger logger = Logger.getLogger("");
740 		logger.setLevel(Level.ALL);
741 		
742 		Handler []handler = logger.getHandlers();
743 		for(int i = 0; i < logger.getHandlers().length; i++) 
744 			logger.removeHandler(handler[i]);
745 
746 		
747 		//Write to a file if a log file was specified
748 		if(logFile != null) {
749 			FileHandler file = setupFileHandler(logFile);
750 			logger.addHandler(file);
751 			logHandlers.add(file);
752 		}
753 		//Write to standard out if the log option flag was specified
754 		else if (showLog == true && rptFile == null) {
755 			StreamHandler stream = setupStreamHandler(System.out);
756 			logger.addHandler(stream);
757 			logHandlers.add(stream);
758 		}
759 		//Write to both standard out and memory if the log option flag
760 		//was specified and report file was specified
761 		else if (showLog == true && rptFile != null) {
762 			byteStream = new ByteArrayOutputStream();
763 			StreamHandler byteArray = setupStreamHandler(byteStream);
764 			logger.addHandler(byteArray);
765 			logHandlers.add(byteArray);
766 			
767 			StreamHandler stream = setupStreamHandler(System.out);
768 			logger.addHandler(stream);
769 			logHandlers.add(stream);
770 		}
771 		//Write to memory if none of the above conditions exist
772 		else {
773 			byteStream = new ByteArrayOutputStream();
774 			StreamHandler byteArray = setupStreamHandler(byteStream);
775 			logger.addHandler(byteArray);
776 			logHandlers.add(byteArray);
777 		}
778 			
779 	}
780 	
781 	/***
782 	 * Sets up a handler to an outputstream.
783 	 * @param out An outputstream to write the logger to
784 	 * @return a stream handler
785 	 */
786 	private StreamHandler setupStreamHandler(OutputStream out) {
787 		StreamHandler stream = new StreamHandler(out, new ToolsLogFormatter());
788 		stream.setLevel(Level.ALL);
789 		
790 		return stream;
791 	}
792 	
793 	/***
794 	 * Sets up a file handler.
795 	 * @param log The name of the file to write the logger to
796 	 * @return a file handler to the input file
797 	 */
798 	private FileHandler setupFileHandler(File log) 
799 	               throws ApplicationException, SystemException {
800 		FileHandler file = null;
801 		try {
802 			file = new FileHandler(log.toString(), false);
803 			file.setLevel(Level.ALL);
804 			file.setFormatter(new ToolsLogFormatter());
805 		} catch (SecurityException s) {
806 			throw new SystemException(s.getMessage());
807 		} catch (IOException iEx) {
808 			throw new ApplicationException(iEx.getMessage());
809 		}
810 		return file;
811 	}
812 	
813 	/***
814 	 * Parse the dictionary files.
815 	 * 
816 	 * @param dictionary a list of dictionary URLs
817 	 * @return a Dictionary object that includes all the dictionary
818 	 *  information from all the dictionary files passed in.
819 	 * @throws ApplicationException
820 	 * @throws gov.nasa.pds.tools.label.parser.ParseException 
821 	 * @throws UnknownLabelStatusException 
822 	 */
823 	public Dictionary readDictionaries(List dictionary) throws
824 	                          ApplicationException {
825 		Dictionary dict = null;
826 		URL url = null;
827 		Iterator i = dictionaries.iterator();
828 		
829 		try {
830 			url = (URL) i.next();
831 			dict = DictionaryParser.parse(url, alias);
832             logMessage(Level.CONFIG, "Dictionary version        \n"
833             		           + dict.getInformation(), url.toString());
834             //Log dictionary status only upon a FAIL
835             if(dict.getStatus().equals(FAIL)) {
836             	throw new gov.nasa.pds.tools.label.parser.ParseException(
837             			"Dictionary parsing failed: " + url.toString());
838             } 
839 			//Parse the rest of the dictionaries
840 			while (i.hasNext()) {
841 				url = (URL) i.next();
842 				Dictionary mergeDict = DictionaryParser.parse(url, alias);
843 				dict.merge(mergeDict, true);
844 				logMessage(Level.CONFIG, "Dictionary version        \n"
845 					+ mergeDict.getInformation(), url.toString());
846 				//Log dictionary status only upon a FAIL
847 				if(mergeDict.getStatus().equals(FAIL)) {
848 	            	throw new gov.nasa.pds.tools.label.parser.ParseException(
849 	            			"Dictionary parsing failed: " + url.toString());
850 				}
851 			}
852 		} catch (IOException iEx) {
853 			throw new ApplicationException(iEx.getMessage());
854 		} catch (gov.nasa.pds.tools.label.parser.ParseException pe) {
855 			logMessage(ToolsLevel.NOTIFICATION, "FAIL", url.toString());
856 			dictionaryPassed = false;
857 		}
858 		return dict;
859 	}
860 	
861 	/***
862 	 * Sets up the include paths and flag to follow pointers in
863 	 * the parser.
864 	 * @param parser The label parser to set properties for
865 	 * @throws ApplicationException 
866 	 */
867 	private void setParserProps(LabelParser parser) 
868 	                                  throws ApplicationException {
869 		URL path = null;
870 		if (includePaths != null) {
871 			for( Iterator i = includePaths.iterator(); i.hasNext(); ) {
872 				try {
873 					path = Utility.toURL(i.next().toString());
874 					path.openStream().close();
875 				} catch (Exception eX) {
876 					throw new ApplicationException(eX.getMessage());
877 				}
878 				parser.addIncludePath(path);
879 			}
880 		}
881 
882 		if (followPtrs == false)
883 			parser.getProperties().setProperty("parser.pointers", "false");
884 	}
885 	
886 	/***
887 	 * Processes a target.
888 	 * 
889 	 * @param target The file or URL to process
890 	 * @param getSubDirs 'True' to look for sub-directories, 'false' otherwise
891 	 * @return A FileList object containing the sub-directories and files
892 	 *  found in the target, if any
893 	 * @throws ApplicationException 
894 	 * @throws SystemException 
895 	 */
896 	public FileList processTarget(String target, boolean getSubDirs) 
897 						throws ApplicationException, SystemException {		
898 		FileListGenerator fileGen = new FileListGenerator();
899 		FileList list = new FileList();
900 		fileGen.setFilters(regexp, noFiles, noDirs);
901 		
902 		try {
903 			list = fileGen.visitTarget(target, getSubDirs);
904 		}catch(IOException io) {
905 			throw new ApplicationException(io.getMessage());
906 		}catch (Exception eX) {
907 			throw new SystemException(eX.getMessage());
908 		}
909 		return list;
910 	}
911 	
912 	/***
913 	 * Validate labels, performing only syntactic validation.
914 	 * 
915 	 * @param targets
916 	 * @return A ValidationRecord containing the results of the validation
917 	 * run.
918 	 * @throws MalformedURLException
919 	 * @throws ApplicationException
920 	 * @throws SystemException
921 	 * @throws UnknownLabelStatusException
922 	 */
923 	public ValidationRecord validateLabels(List targets) 
924 	        throws MalformedURLException, ApplicationException,
925 	               SystemException, UnknownLabelStatusException {
926 		return validateLabels(targets, null);
927 	}
928 	
929 	/***
930 	 * Validate labels, performing both syntactic and semantic validation.
931 	 * If the target is a directory, this method will validate all labels
932 	 * found within the directory and its sub-directories. If recursion
933 	 * is turned OFF, then sub-directories will not be looked into.
934 	 * 
935 	 * @param targets a list of files, directories, and/or URLs.
936 	 * @param dict the dictionary file.
937 	 * @return A ValidationRecord containing the results of the validation.
938 	 * @throws SystemException 
939 	 * @throws ApplicationException 
940 	 * @throws MalformedURLException 
941 	 * @throws UnknownLabelStatusException 
942 	 */	
943 	public ValidationRecord validateLabels(List targets, Dictionary dict) 
944 	            throws ApplicationException, SystemException, 
945 	                   MalformedURLException, UnknownLabelStatusException {
946 		
947 		ValidationRecord record = new ValidationRecord();
948 		for (Iterator i1 = targets.iterator(); i1.hasNext();) {
949 			FileList fileList = new FileList();
950 			fileList = processTarget(i1.next().toString(), recursive);		
951 			for (Iterator i2 = fileList.getFiles().iterator(); i2.hasNext();) {
952 				URL target = Utility.toURL(i2.next().toString());				
953 				record.add(validateLabel(target, dict));
954 			}		
955 			if (!fileList.getDirs().isEmpty())
956 				record.add(validateLabels(fileList.getDirs(), dict));
957 		}
958 		return record;
959 	}
960 	
961 	/***
962 	 * Validate a label file.
963 	 * 
964 	 * @param file The URL of the file to be validated
965 	 * @param dict a Dictionary object needed for semantic validation. If null,
966 	 *             only syntactic validation will be performed.
967 	 * @return A ValidationRecord object containing the results of the validation.
968 	 * @throws ApplicationException 
969 	 * @throws SystemException 
970 	 * @throws UnknownLabelStatusException 
971 	 */	
972 	public ValidationRecord validateLabel(URL url, Dictionary dict) 
973 						        throws ApplicationException, SystemException,
974 						                         UnknownLabelStatusException {		
975 		LabelParserFactory factory = LabelParserFactory.getInstance();
976 		LabelParser parser = factory.newLabelParser();
977 		Label label = null;
978 		setParserProps(parser);
979 		
980 		if (progress)
981 			showProgress(url);
982 		
983 		ValidatorFactory vf = ValidatorFactory.getInstance();
984 		Validator validator = vf.newInstance(parser, url, force);
985 		try {
986 			if(dict == null)
987 				label = validator.validate(url);
988 			else
989 				label = validator.validate(url, dict);
990 		} catch (gov.nasa.pds.tools.label.parser.ParseException pEx) {
991 			logMessage(ToolsLevel.NOTIFICATION, "SKIP", url.toString());
992 			return new ValidationRecord("SKIP", 1, 0);
993 		} catch (IOException iEx) {
994 			logMessage(ToolsLevel.NOTIFICATION, "SKIP", url.toString());
995 			logMessage(ToolsLevel.WARNING, iEx.getMessage(), url.toString());
996 			return new ValidationRecord("SKIP", 1, 0);
997 		}
998 		logMessage(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString());	
999 		return new ValidationRecord(label.getStatus(), label.getNumWarnings(), label.getNumErrors());
1000 	}
1001 	
1002 	/***
1003 	 *  Prints out the current directory being validated and
1004 	 *  represents each file being validated by an asterisk.
1005 	 *  This is used when progress reporting is enabled.
1006 	 * 
1007 	 * @param file The URL being validated
1008 	 * @throws SystemException 
1009 	 */
1010 	public void showProgress(URL file) throws SystemException {
1011 		URL dir = null;
1012 		try {
1013 			dir = new URL(file.toString().
1014 						substring(0, file.toString().lastIndexOf("/")));
1015 		} catch (MalformedURLException eX) {
1016 			throw new SystemException(eX.getMessage());
1017 		}
1018 
1019 		if (dir.toString().equals(currDir)) {
1020 			System.err.print(FILE_REP);
1021 		}
1022 		else {
1023 			currDir = new String(dir.toString());
1024 			System.err.println("\nValidating file(s) in: " + currDir);
1025 			System.err.print(FILE_REP);
1026 		}
1027 	}
1028 	
1029 	/***
1030 	 * Creates a human-readable report.
1031 	 * 
1032 	 * @param log Where the xml log is located. If null, then the xml log
1033 	 *  will be read from memory.
1034 	 * @param report Where the human-readable report will be written to.
1035 	 *  If null, then it goes to standard out.
1036 	 * @param level The severity level to include in the report. Can be
1037 	 *  "INFO", "WARNING", or "ERROR".
1038 	 * @param style The reporting style to generate. Can be either "full",
1039 	 *  "sum", or "min" for a full, summary, or minimal report, respectively.
1040 	 * @throws SystemException 
1041 	 * @throws ApplicationException 
1042 	 */
1043 	public void doReporting(File log, File report, String level, String style) 
1044 	                   throws SystemException, ApplicationException {
1045 		InputStream input = null;
1046 		OutputStream output = null;	
1047 		try {
1048 			input = getRptInStream(log);
1049 			output = getRptOutStream(report);
1050 		} catch (FileNotFoundException ex) {
1051 			throw new ApplicationException(ex.getMessage());
1052 		}	
1053 		String stylesheet = getStyleSheet(style);
1054 		Report humanReport = new Report(input, stylesheet);
1055 		try {
1056 			humanReport.generateReport(output, level);
1057 		} catch (TransformerException t) {
1058 			throw new SystemException(t.getMessage());
1059 		}
1060 		
1061 		//If report is written to standard out, add an extra line terminator
1062 		if(report == null)
1063 			System.out.println();
1064 	}
1065 	
1066 	/***
1067 	 * Gets the proper XSL stylesheet file to use when transforming
1068 	 * the log into a human-readable report.
1069 	 * 
1070 	 * @param style The level of detail for the reporting. The valid values
1071 	 * are 'full' for a full view, 'sum' for a summary view, and 'min' for a 
1072 	 * minimal view.
1073 	 * 
1074 	 * @return The XSL stylesheet file name.
1075 	 * @throws ApplicationException
1076 	 */
1077 	public String getStyleSheet(String style) throws ApplicationException {
1078 		if (style.equalsIgnoreCase("full"))
1079 			return FULLXSL;
1080 		else if (style.equalsIgnoreCase("sum"))
1081 			return SUMXSL;
1082 		else if (style.equalsIgnoreCase("min"))
1083 			return MINXSL;
1084 		else {
1085 			throw new ApplicationException("Invalid style specified: "
1086 				+ style + ". Must be 'full', 'sum' or 'min'");
1087 		}
1088 	}
1089 	
1090 	/***
1091 	 * Returns the input stream of the XML log for the final report.
1092 	 * 
1093 	 * @param file A file name. If null, then a ByteArrayInputStream is returned. 
1094 	 * 
1095 	 * @return The proper input stream 
1096 	 * @throws FileNotFoundException 
1097 	 */
1098 	private InputStream getRptInStream(File file) throws FileNotFoundException {	
1099 		if(file != null)
1100 				return new FileInputStream(file);
1101 		else
1102 			return new ByteArrayInputStream(byteStream.toByteArray());
1103 	}
1104 	
1105 	/***
1106 	 * Returns the appropriate output stream for the final report.
1107 	 * @param rpt The report file name. If null, then the standard out
1108 	 *  stream is returned.
1109 	 * 
1110 	 * @return The proper output stream
1111 	 * @throws FileNotFoundException
1112 	 */
1113 	private OutputStream getRptOutStream(File rpt) throws FileNotFoundException {
1114 		if(rpt != null) 
1115 			return new FileOutputStream(rpt);
1116 		else
1117 			return System.out;
1118 	}
1119 	  
1120     /***
1121      * Closes the handlers that were set for the logger.
1122      *
1123      */
1124 	public void closehandle() {
1125 		for(Iterator i=logHandlers.iterator(); i.hasNext();) {
1126 			Handler handler = (Handler) i.next();
1127 			handler.close();
1128 		}
1129 	}
1130 	
1131 	/***
1132 	 * Implementation to perform automated PDS validation.
1133 	 * 
1134 	 * <p>
1135 	 * The main calls the following methods (in this order):
1136 	 * <p>
1137 	 * To setup the flags and parse the command-line options:
1138 	 * <ul> 
1139 	 * <li>parseLine</li>
1140 	 * <li>queryCmdLine</li>
1141 	 * </ul>
1142 	 * 
1143 	 * <p>
1144 	 * To setup the logger and log the report header information:
1145 	 * <ul>
1146 	 * <li>setupLogger</li>
1147 	 * <li>logRptHeader</li>
1148 	 * </ul>
1149 	 * 
1150 	 * <p>
1151 	 * To perform validation:
1152 	 * <ul>
1153 	 * <li>readDictionaries (if the PSDD was passed in)</li>
1154 	 * <li>validateLabels</li>
1155 	 * </ul>
1156 	 * 
1157 	 * <p>
1158 	 * To create the final report:
1159 	 * <ul>
1160 	 * <li>closehandle (to flush the buffers)</li>
1161 	 * <li>doReporting</li>
1162 	 * </ul>
1163 	 * 
1164 	 * <p>
1165 	 * Reporting is not generated if the log flag was specified with
1166 	 * no file spec and the report file flag was not specified
1167 	 * 
1168 	 * <p>
1169 	 * VTool returns an appropriate exit status based on validation results.
1170 	 * 
1171 	 * @param argv Arguments passed on the command-line
1172 	 */
1173 	public static void main(String[] argv) {
1174 		VTool vtool = new VTool();
1175 		ExitStatus status = new ExitStatus();
1176 
1177 		if (argv.length == 0) {
1178 			System.out.println("\nType 'VTool -h' for usage");
1179 			System.exit(SUCCESS);
1180 		}
1181 
1182 		try {
1183 			//Parse the command line
1184 			CommandLine commandLine = vtool.parseLine(argv);
1185 			// Query the command line
1186 			vtool.queryCmdLine(commandLine);
1187 			//Setup the logger
1188 			vtool.setupLogger();
1189 			// Log the report header
1190 			vtool.logRptHeader();	
1191 			
1192 			ValidationRecord record = new ValidationRecord();
1193 			if ( vtool.dictionaries.isEmpty() ) {
1194 				record = vtool.validateLabels(vtool.targets);
1195 			    status.setStatus(record);
1196 			}
1197 			else {
1198 				//If dictionary parsing fails, VTool will not
1199 				//perform the validation.
1200 				Dictionary dictionary = vtool.readDictionaries(vtool.dictionaries);
1201                 if(vtool.dictionaryPassed) {
1202                 	record = vtool.validateLabels(vtool.targets, dictionary);
1203                 	status.setStatus(record);
1204                 }
1205                 else
1206                 	status.setStatus(APPLICATION_ERROR);
1207 			}
1208 			vtool.closehandle();
1209 		
1210 			//Add extra line terminator if progress reporting was enabled
1211 			if(vtool.progress == true)
1212 				System.err.println();
1213 		
1214 			//If the log flag with no file spec was specified AND the
1215 			//report file flag is not specified, then do not create a report
1216 			if( !(vtool.showLog && vtool.rptFile == null) ) {
1217 				vtool.doReporting(vtool.logFile, vtool.rptFile, 
1218 						          vtool.severity, vtool.rptStyle);
1219 			}
1220 		} catch(ApplicationException app) {
1221 			System.err.println(app.getMessage());
1222 			status.setStatus(APPLICATION_ERROR);
1223 		} catch(Exception e) {
1224 			System.err.println(e.getMessage());
1225 			status.setStatus(SYSTEM_ERROR);
1226 		} finally {
1227 			System.exit(status.getStatus(vtool.severity));
1228 		}
1229 	}
1230 }