View Javadoc

1   // Copyright 2006-2007, by the California Institute of Technology.
2   // ALL RIGHTS RESERVED. United States Government Sponsorship acknowledged.
3   // Any commercial use must be negotiated with the Office of Technology Transfer
4   // at the California Institute of Technology.
5   //
6   // This software is subject to U. S. export control laws and regulations
7   // (22 C.F.R. 120-130 and 15 C.F.R. 730-774). To the extent that the software
8   // is subject to U.S. export control laws and regulations, the recipient has
9   // the responsibility to obtain export licenses or other export authority as
10  // may be required before exporting such information to foreign countries or
11  // providing access to foreign nationals.
12  //
13  // $Id: DefaultLabelParser.java 3836 2009-01-14 18:50:02Z pramirez $ 
14  //
15  
16  package gov.nasa.pds.tools.label.parser;
17  
18  import gov.nasa.pds.tools.dict.Dictionary;
19  import gov.nasa.pds.tools.dict.parser.DictionaryParser;
20  import gov.nasa.pds.tools.dict.type.UnsupportedTypeException;
21  import gov.nasa.pds.tools.label.AttributeStatement;
22  import gov.nasa.pds.tools.label.GroupStatement;
23  import gov.nasa.pds.tools.label.Label;
24  import gov.nasa.pds.tools.label.MalformedSFDULabel;
25  import gov.nasa.pds.tools.label.ObjectStatement;
26  import gov.nasa.pds.tools.label.SFDULabel;
27  import gov.nasa.pds.tools.label.Statement;
28  import gov.nasa.pds.tools.label.antlr.ODLLexer;
29  import gov.nasa.pds.tools.label.antlr.ODLParser;
30  import gov.nasa.pds.tools.label.validate.CountListener;
31  import gov.nasa.pds.tools.label.validate.DefinitionNotFoundException;
32  import gov.nasa.pds.tools.label.validate.ElementValidator;
33  import gov.nasa.pds.tools.label.validate.GroupValidator;
34  import gov.nasa.pds.tools.label.validate.LabelValidator;
35  import gov.nasa.pds.tools.label.validate.ObjectValidator;
36  import gov.nasa.pds.tools.label.validate.Status;
37  import gov.nasa.pds.tools.logging.ToolsLevel;
38  import gov.nasa.pds.tools.logging.FullLogFormatter;
39  import gov.nasa.pds.tools.logging.ToolsLogRecord;
40  
41  import java.io.BufferedReader;
42  import java.io.IOException;
43  import java.io.InputStream;
44  import java.io.InputStreamReader;
45  import java.net.URL;
46  import java.util.ArrayList;
47  import java.util.Collections;
48  import java.util.Iterator;
49  import java.util.List;
50  import java.util.Properties;
51  import java.util.logging.Handler;
52  import java.util.logging.Level;
53  import java.util.logging.Logger;
54  import java.util.logging.StreamHandler;
55  
56  import antlr.ANTLRException;
57  
58  /***
59   * Default implementation
60   * @author pramirez
61   * @version $Revision: 3836 $
62   * 
63   */
64  public class DefaultLabelParser implements LabelParser, Status {
65      private static Logger log = Logger.getLogger(DefaultLabelParser.class.getName());
66      private Properties properties;
67      private List includePaths;
68      private List labelValidators;
69      private List fragmentValidators;
70      
71      public DefaultLabelParser() {
72          properties = new Properties();
73          includePaths = new ArrayList();
74          labelValidators = new ArrayList();
75          fragmentValidators = new ArrayList();
76      }
77  
78      /* (non-Javadoc)
79       * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL)
80       */
81      public Label parse(URL url) throws ParseException, IOException {
82          Label label = null;
83          log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString()));
84          try {
85              label = parseLabel(url);
86          } catch (ParseException pe) {
87          	log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
88          	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
89          	throw pe;
90          } catch (IOException ioe) {
91          	log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
92          	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
93          	throw ioe;
94          }
95          
96          log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
97          
98          return label;
99      }
100     
101     private Label parseLabel(URL url) throws ParseException, IOException {
102     	Label label = null;
103         
104         //Not all streams can support marking so stream will be open multiple times to look for header
105         //First time to process the SFDUs
106         InputStream sfduCheck = url.openStream();
107         
108         List sfdus = consumeSFDUHeader(sfduCheck);
109         for (Iterator i = sfdus.iterator(); i.hasNext();) {
110             log.log(new ToolsLogRecord(Level.INFO, "Found SFDU Label: " + i.next().toString(), url.toString()));
111         }
112         
113         //On the next input stream we will need to skip 20 bytes for every SFDULabel
114         int skip = sfdus.size()*20;
115         //Also add 2 for carriage return line feed if there is a header
116         if (skip != 0)
117             skip += 2;
118         
119         sfduCheck.close();
120         
121         InputStream pdsCheck = url.openStream();
122         //Now look for PDS_VERSION_ID to ensure that this is a file we want to validate
123         BufferedReader reader = new BufferedReader(new InputStreamReader(pdsCheck));
124         reader.skip(skip);
125         String version = reader.readLine();
126         String[] line = null;
127         if (version != null) {
128             version = version.trim();
129             line = version.split("=");  
130         }
131         
132         if (version == null || line == null || line.length != 2) {
133             log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
134             label = new Label();
135             label.setStatus(Label.SKIP);
136             label.incrementWarnings();
137             return label;
138         }
139         
140         String name = line[0].trim();
141         String value = line[1].trim();
142           
143         if (!"PDS_VERSION_ID".equals(name)) {
144             log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
145             label = new Label();
146             label.setStatus(Label.SKIP);
147             label.incrementWarnings();
148             return label;
149         }
150         
151         pdsCheck.close();
152         log.log(new ToolsLogRecord(Level.INFO, "Parsing label with PDS_VERSION_ID = " + value, url.toString()));
153         
154         InputStream input = url.openStream();
155         input.skip(skip);
156         ODLLexer lexer = new ODLLexer(input);
157         lexer.setFilename(url.toString());
158         ODLParser parser = new ODLParser(lexer);
159         parser.setFilename(url.toString());
160         parser.setFollowPointers(Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue());
161         
162         if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
163             URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
164             addIncludePath(base);
165             parser.setIncludePaths(includePaths);
166         } else {
167             log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString()));
168         }
169         
170         try {
171             label = parser.label();
172             label.setStatus(PASS);
173             label.setStatus(lexer.getStatus());
174             label.incrementErrors(lexer.getNumErrors());
175             label.incrementWarnings(lexer.getNumWarnings());
176             label.setStatus(parser.getStatus());
177             label.incrementErrors(parser.getNumErrors());
178             label.incrementWarnings(parser.getNumWarnings());
179         } catch (ANTLRException ex) {
180             label.setStatus(FAIL);
181             label.incrementErrors();
182             log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString()));
183             throw new ParseException(ex.getMessage());
184         }
185         
186         log.log(new ToolsLogRecord(Level.INFO, "Finished parsing", url.toString()));
187         
188         CountListener listener = new CountListener();
189         for (Iterator i = labelValidators.iterator(); i.hasNext();) {
190             LabelValidator validator = (LabelValidator) i.next();
191             boolean valid = validator.isValid(label, listener);
192             if (!valid)
193                 label.setStatus(FAIL);
194         }
195         label.incrementErrors(listener.getNumErrors());
196         label.incrementWarnings(listener.getNumWarnings());
197 
198         return label;
199     }
200     
201     private List consumeSFDUHeader(InputStream input) throws IOException {
202         List sfdus = new ArrayList();
203         
204         byte[] sfduLabel = new byte[20];
205         int count = input.read(sfduLabel);
206         if (count == 20) {
207             try {
208                 SFDULabel sfdu = new SFDULabel(sfduLabel);
209                 if ("CCSD".equals(sfdu.getControlAuthorityId())) {
210                     sfdus.add(sfdu);
211                     //Read in second SFDU label
212                     input.read(sfduLabel);
213                     sfdus.add(new SFDULabel(sfduLabel));
214                 }
215             } catch (MalformedSFDULabel e) {
216                 //For now we can ignore this error as there is likely not a header.
217             }
218             
219         }
220         
221         return sfdus;
222     }
223 
224     /* (non-Javadoc)
225      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL, gov.nasa.jpl.pds.tools.dict.Dictionary)
226      */
227     public Label parse(URL url, Dictionary dictionary) throws ParseException, IOException {
228         Label label = null;
229 
230         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString()));
231         try {
232 	        //First parse the file and get back the label object
233 	        label = parseLabel(url);
234         } catch (ParseException pe) {
235         	log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
236         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
237         	throw pe;
238         } catch (IOException ioe) {
239         	log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
240         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
241         	throw ioe;
242         }
243         
244         if (label != null && !label.getStatus().equals(Label.SKIP)) {
245 	        log.log(new ToolsLogRecord(Level.INFO, "Starting semantic validation.", url.toString()));
246 	        label = semanticCheck(url, dictionary, label);
247 	        log.log(new ToolsLogRecord(Level.INFO, "Finished semantic validation.", url.toString()));
248         }
249         
250         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
251         
252         return label;
253     }
254     
255     private Label semanticCheck(URL url, Dictionary dictionary, Label label) {
256     	//Check all the statements
257         List statements = label.getStatements();
258         Collections.sort(statements);
259         CountListener listener = new CountListener();
260         for (Iterator i = statements.iterator(); i.hasNext();) {
261             Statement statement = (Statement) i.next();
262             if (statement instanceof AttributeStatement) {
263                 if (!ElementValidator.isValid(dictionary, (AttributeStatement) statement, listener)) {
264                     label.setStatus(FAIL);
265                 }
266             } else if (statement instanceof ObjectStatement) {
267                 if (!ObjectValidator.isValid(dictionary, (ObjectStatement) statement, listener))
268                     label.setStatus(FAIL);
269             } else if (statement instanceof GroupStatement) {
270                 if (!GroupValidator.isValid(dictionary, (GroupStatement) statement, listener))
271                     label.setStatus(FAIL);
272             }
273         }
274         label.incrementErrors(listener.getNumErrors());
275         label.incrementWarnings(listener.getNumWarnings());
276         
277     	return label;
278     }
279 
280     /* (non-Javadoc)
281      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL, gov.nasa.jpl.pds.tools.dict.Dictionary, boolean)
282      */
283     public Label parse(URL file, Dictionary dictionary, boolean dataObjectValidation)  throws ParseException, IOException {
284         return parse(file, dictionary);
285     }
286 
287     /* (non-Javadoc)
288      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#setProperties(java.util.Properties)
289      */
290     public void setProperties(Properties properties) {
291         this.properties.putAll(properties);
292     }
293 
294     /* (non-Javadoc)
295      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getProperties()
296      */
297     public Properties getProperties() {
298         return properties;
299     }
300 
301     /* (non-Javadoc)
302      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getPDSVersion()
303      */
304     public String getPDSVersion() {
305         return "PDS3";
306     }
307 
308     /* (non-Javadoc)
309      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getODLVersion()
310      */
311     public String getODLVersion() {
312         return "2.1";
313     }
314     
315     /* (non-Javadoc)
316      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, boolean)
317      */
318     public Label parsePartial(URL url) throws ParseException, IOException {
319         Label label =  null;
320         
321         try {
322         	label = parsePartial(null, url);
323         } catch (ParseException pe) {
324         	log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
325         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
326         	throw pe;
327         } catch (IOException ioe) {
328         	log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
329         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
330         	throw ioe;
331         }
332         
333         CountListener listener = new CountListener();
334         for (Iterator i = fragmentValidators.iterator(); i.hasNext();) {
335             LabelValidator validator = (LabelValidator) i.next();
336             boolean valid = validator.isValid(label, listener);
337             if (!valid)
338                 label.setStatus(FAIL);
339         }
340         label.incrementErrors(listener.getNumErrors());
341         label.incrementWarnings(listener.getNumWarnings());
342         
343         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
344         
345         return label;
346     }
347     
348     /***
349      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(String,java.net.URL, boolean)
350      */
351     public Label parsePartial(String context, URL url) throws ParseException, IOException {
352         Label label = null;
353         
354         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString(), context));
355         log.log(new ToolsLogRecord(ToolsLevel.INFO, "Parsing label fragment", url.toString(), context));
356         
357         //Not all streams can support marking so stream will be open multiple times to look for header
358         //First time to process the SFDUs
359         InputStream sfduCheck = url.openStream();
360         
361         List sfdus = consumeSFDUHeader(sfduCheck);
362         
363         //On the next input stream we will need to skip 20 bytes for every SFDULabel
364         int skip = sfdus.size()*20;
365         //Also add 2 for carriage return line feed if there is a header
366         if (skip != 0)
367             skip += 2;
368         
369         sfduCheck.close();
370         if (sfdus.size() > 0) {
371             log.log(new ToolsLogRecord(Level.WARNING, "Label fragments should not contain SFDU headers."));
372         }
373         
374         InputStream input = url.openStream();
375         input.skip(skip);
376         ODLLexer lexer = new ODLLexer(input);
377         lexer.setFilename(url.toString());
378         lexer.setContext(context);
379         ODLParser parser = new ODLParser(lexer);
380         parser.setFilename(url.toString());
381         parser.setContext(context);
382         parser.setFollowPointers(Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue());
383         
384         if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
385             URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
386             addIncludePath(base);
387             parser.setIncludePaths(includePaths);
388         } else {
389             log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString(), context));
390         }
391         
392         try {
393             label = parser.label();
394             label.setStatus(PASS);
395             label.setStatus(lexer.getStatus());
396             label.incrementErrors(lexer.getNumErrors());
397             label.incrementWarnings(lexer.getNumWarnings());
398             label.setStatus(parser.getStatus());
399             label.incrementErrors(parser.getNumErrors());
400             label.incrementWarnings(parser.getNumWarnings());
401         } catch (ANTLRException ex) {
402             label.setStatus(FAIL);
403             label.incrementErrors();
404             log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString(), context));
405             throw new ParseException(ex.getMessage());
406         }
407         
408         if (label.getStatement("PDS_VERSION_ID") != null) {
409             label.incrementWarnings();
410             log.log(new ToolsLogRecord(Level.WARNING, "Fragment contains PDS_VERSION_ID which should not be present in a label fragment.", url.toString(), context));
411         }
412 
413         log.log(new ToolsLogRecord(ToolsLevel.INFO, "Finished parsing label fragment", url.toString(), context));
414         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "END", url.toString(), context));
415         
416         return label;
417     }
418 
419     /* (non-Javadoc)
420      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, gov.nasa.pds.tools.dict.Dictionary)
421      */
422     public Label parsePartial(URL url, Dictionary dictionary) throws ParseException, IOException {
423     	return parsePartial(url, dictionary, false);
424     }
425     
426     /* (non-Javadoc)
427      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, gov.nasa.pds.tools.dict.Dictionary, boolean)
428      */
429     public Label parsePartial(URL url, Dictionary dictionary, boolean skipVersion) throws ParseException, IOException {
430     	Label label =  null;
431         
432         try {
433         	label = parsePartial(null, url);
434         } catch (ParseException pe) {
435         	log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
436         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
437         	throw pe;
438         } catch (IOException ioe) {
439         	log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
440         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
441         	throw ioe;
442         }
443         
444         CountListener listener = new CountListener();
445         for (Iterator i = fragmentValidators.iterator(); i.hasNext();) {
446             LabelValidator validator = (LabelValidator) i.next();
447             boolean valid = validator.isValid(label, listener);
448             if (!valid)
449                 label.setStatus(FAIL);
450         }
451         label.incrementErrors(listener.getNumErrors());
452         label.incrementWarnings(listener.getNumWarnings());
453         
454     	log.log(new ToolsLogRecord(Level.INFO, "Starting semantic validation.", url.toString()));
455         label = semanticCheck(url, dictionary, label);
456         log.log(new ToolsLogRecord(Level.INFO, "Finished semantic validation.", url.toString()));
457 
458         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
459     	
460     	return label;
461     }
462 
463     
464     /***
465      * 
466      * @param args
467      * @throws Exception
468      */
469     public static void main(String [] args) throws Exception {
470         Logger logger = Logger.getLogger("");
471         
472         Handler [] handler = logger.getHandlers();
473         for (int i = 0; i < logger.getHandlers().length; i++)
474             logger.removeHandler(handler[i]);
475         
476         StreamHandler stream = new StreamHandler(System.out, new FullLogFormatter());
477         logger.addHandler(stream);
478         logger.setLevel(Level.ALL);
479         
480         LabelParserFactory factory = LabelParserFactory.getInstance();
481         LabelParser parser = factory.newLabelParser();
482         Label label = null;
483         URL labelURL = null;
484         URL dictionaryURL = null;
485         URL includePathURL = null;
486         Boolean pointers = null; 
487         Boolean aliasing = new Boolean(true);
488         
489         if (args.length%2 == 0) {
490             for (int i=0; i<args.length; i+=2) {
491                 if (args[i].equals("--label") || args[i].equals("--l"))
492                     labelURL = new URL(args[i+1]);
493                 else if (args[i].equals("--dictionary") || args[i].equals("--d"))
494                     dictionaryURL = new URL(args[i+1]);
495                 else if (args[i].equals("--include") || args[i].equals("--i"))
496                     includePathURL = new URL(args[i+1]);
497                 else if (args[i].equals("--pointers") || args[i].equals("--p"))
498                     pointers = Boolean.valueOf(args[i+1]);
499                 else if (args[i].equals("--aliasing") || args[i].equals("--a"))
500                     aliasing = Boolean.valueOf(args[i+1]);
501                 else {
502                     System.out.println("Invalid flag " + args[i]);
503                     System.exit(1);
504                 }
505             }
506         }
507         
508         if (pointers != null) {
509             parser.getProperties().setProperty("parser.pointers", pointers.toString());
510         }
511         
512         if (includePathURL != null)
513             parser.addIncludePath(includePathURL);
514         
515         Logger logFile = Logger.getLogger(DefaultLabelParser.class.getName());
516         if (dictionaryURL == null) {
517             label = parser.parse(labelURL);
518             System.out.println("Errors: " + label.getNumErrors());
519             System.out.println("Warnings: " + label.getNumWarnings());
520         } else {
521             Dictionary dictionary = DictionaryParser.parse(dictionaryURL, aliasing.booleanValue());
522             logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, dictionary.getStatus(), dictionaryURL.toString()));
523             label = parser.parse(labelURL, dictionary);
524             System.out.println("Errors: " + label.getNumErrors());
525             System.out.println("Warnings: " + label.getNumWarnings());
526         }
527         
528         stream.flush();
529         stream.close();
530     }
531 
532     /* (non-Javadoc)
533      * @see gov.nasa.pds.tools.label.parser.LabelParser#addIncludePath(java.net.URL)
534      */
535 	public void addIncludePath(URL includePath) {
536         if (!includePaths.contains(includePath))
537 		   includePaths.add(includePath);
538 	}
539 
540     /* (non-Javadoc)
541      * @see gov.nasa.pds.tools.label.parser.LabelParser#addValidator(gov.nasa.pds.tools.label.validate.LabelValidator)
542      */
543     public void addLabelValidator(LabelValidator validator) {
544         labelValidators.add(validator);
545     }
546     
547     public void addFragmentValidator(LabelValidator validator) {
548     	fragmentValidators.add(validator);
549     }
550 
551 }