View Javadoc

1   // Copyright 2006-2007, by the California Institute of Technology.
2   // ALL RIGHTS RESERVED. United States Government Sponsorship acknowledged.
3   // Any commercial use must be negotiated with the Office of Technology Transfer
4   // at the California Institute of Technology.
5   //
6   // This software is subject to U. S. export control laws and regulations
7   // (22 C.F.R. 120-130 and 15 C.F.R. 730-774). To the extent that the software
8   // is subject to U.S. export control laws and regulations, the recipient has
9   // the responsibility to obtain export licenses or other export authority as
10  // may be required before exporting such information to foreign countries or
11  // providing access to foreign nationals.
12  //
13  // $Id: DefaultLabelParser.java 2652 2007-04-30 15:18:41Z mcayanan $ 
14  //
15  
16  package gov.nasa.pds.tools.label.parser;
17  
18  import gov.nasa.pds.tools.dict.Dictionary;
19  import gov.nasa.pds.tools.dict.parser.DictionaryParser;
20  import gov.nasa.pds.tools.dict.type.UnsupportedTypeException;
21  import gov.nasa.pds.tools.label.AttributeStatement;
22  import gov.nasa.pds.tools.label.GroupStatement;
23  import gov.nasa.pds.tools.label.Label;
24  import gov.nasa.pds.tools.label.MalformedSFDULabel;
25  import gov.nasa.pds.tools.label.ObjectStatement;
26  import gov.nasa.pds.tools.label.SFDULabel;
27  import gov.nasa.pds.tools.label.Statement;
28  import gov.nasa.pds.tools.label.antlr.ODLLexer;
29  import gov.nasa.pds.tools.label.antlr.ODLParser;
30  import gov.nasa.pds.tools.label.validate.CountListener;
31  import gov.nasa.pds.tools.label.validate.DefinitionNotFoundException;
32  import gov.nasa.pds.tools.label.validate.ElementValidator;
33  import gov.nasa.pds.tools.label.validate.GroupValidator;
34  import gov.nasa.pds.tools.label.validate.LabelValidator;
35  import gov.nasa.pds.tools.label.validate.ObjectValidator;
36  import gov.nasa.pds.tools.label.validate.Status;
37  import gov.nasa.pds.tools.logging.ToolsLevel;
38  import gov.nasa.pds.tools.logging.ToolsLogFormatter;
39  import gov.nasa.pds.tools.logging.ToolsLogRecord;
40  
41  import java.io.BufferedReader;
42  import java.io.IOException;
43  import java.io.InputStream;
44  import java.io.InputStreamReader;
45  import java.net.URL;
46  import java.util.ArrayList;
47  import java.util.Collections;
48  import java.util.Iterator;
49  import java.util.List;
50  import java.util.Properties;
51  import java.util.logging.Handler;
52  import java.util.logging.Level;
53  import java.util.logging.Logger;
54  import java.util.logging.StreamHandler;
55  
56  import antlr.ANTLRException;
57  
58  /***
59   * Default implementation
60   * @author pramirez
61   * @version $Revision: 2652 $
62   * 
63   */
64  public class DefaultLabelParser implements LabelParser, Status {
65      private static Logger log = Logger.getLogger(DefaultLabelParser.class.getName());
66      private Properties properties;
67      private List includePaths;
68      private List labelValidators;
69      private List fragmentValidators;
70      
71      public DefaultLabelParser() {
72          properties = new Properties();
73          includePaths = new ArrayList();
74          labelValidators = new ArrayList();
75          fragmentValidators = new ArrayList();
76      }
77  
78      /* (non-Javadoc)
79       * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL)
80       */
81      public Label parse(URL url) throws ParseException, IOException {
82          Label label = null;
83          log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString()));
84          try {
85              label = parseLabel(url);
86          } catch (ParseException pe) {
87          	log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
88          	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
89          	throw pe;
90          } catch (IOException ioe) {
91          	log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
92          	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
93          	throw ioe;
94          }
95          
96          log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
97          
98          return label;
99      }
100     
101     private Label parseLabel(URL url) throws ParseException, IOException {
102     	Label label = null;
103         
104         //Not all streams can support marking so stream will be open multiple times to look for header
105         //First time to process the SFDUs
106         InputStream sfduCheck = url.openStream();
107         
108         List sfdus = consumeSFDUHeader(sfduCheck);
109         for (Iterator i = sfdus.iterator(); i.hasNext();) {
110             log.log(new ToolsLogRecord(Level.INFO, "Found SFDU Label: " + i.next().toString(), url.toString()));
111         }
112         
113         //On the next input stream we will need to skip 20 bytes for every SFDULabel
114         int skip = sfdus.size()*20;
115         //Also add 2 for carriage return line feed if there is a header
116         if (skip != 0)
117             skip += 2;
118         
119         sfduCheck.close();
120         
121         InputStream pdsCheck = url.openStream();
122         //Now look for PDS_VERSION_ID to ensure that this is a file we want to validate
123         BufferedReader reader = new BufferedReader(new InputStreamReader(pdsCheck));
124         reader.skip(skip);
125         String version = reader.readLine();
126         String[] line = null;
127         if (version != null) {
128             version = version.trim();
129             line = version.split("=");  
130         }
131         
132         if (version == null || line == null || line.length != 2) {
133             log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
134             label = new Label();
135             label.setStatus(Label.SKIP);
136             label.incrementWarnings();
137             return label;
138         }
139         
140         String name = line[0].trim();
141         String value = line[1].trim();
142           
143         if (!"PDS_VERSION_ID".equals(name)) {
144             log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
145             label = new Label();
146             label.setStatus(Label.SKIP);
147             label.incrementWarnings();
148             return label;
149         }
150         
151         pdsCheck.close();
152         log.log(new ToolsLogRecord(Level.INFO, "Parsing label with PDS_VERSION_ID = " + value, url.toString()));
153         
154         InputStream input = url.openStream();
155         input.skip(skip);
156         ODLLexer lexer = new ODLLexer(input);
157         lexer.setFilename(url.toString());
158         ODLParser parser = new ODLParser(lexer);
159         parser.setFilename(url.toString());
160         parser.setFollowPointers(Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue());
161         
162         if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
163             URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
164             addIncludePath(base);
165             parser.setIncludePaths(includePaths);
166         } else {
167             log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString()));
168         }
169         
170         try {
171             label = parser.label();
172             label.setStatus(PASS);
173             label.setStatus(lexer.getStatus());
174             label.incrementErrors(lexer.getNumErrors());
175             label.incrementWarnings(lexer.getNumWarnings());
176             label.setStatus(parser.getStatus());
177             label.incrementErrors(parser.getNumErrors());
178             label.incrementWarnings(parser.getNumWarnings());
179         } catch (ANTLRException ex) {
180             label.setStatus(FAIL);
181             label.incrementErrors();
182             log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString()));
183             throw new ParseException(ex.getMessage());
184         }
185         
186         log.log(new ToolsLogRecord(Level.INFO, "Finished parsing", url.toString()));
187         
188         CountListener listener = new CountListener();
189         for (Iterator i = labelValidators.iterator(); i.hasNext();) {
190             LabelValidator validator = (LabelValidator) i.next();
191             boolean valid = validator.isValid(label, listener);
192             if (!valid)
193                 label.setStatus(FAIL);
194         }
195         label.incrementErrors(listener.getNumErrors());
196         label.incrementWarnings(listener.getNumWarnings());
197 
198         return label;
199     }
200     
201     private List consumeSFDUHeader(InputStream input) throws IOException {
202         List sfdus = new ArrayList();
203         
204         byte[] sfduLabel = new byte[20];
205         int count = input.read(sfduLabel);
206         if (count == 20) {
207             try {
208                 SFDULabel sfdu = new SFDULabel(sfduLabel);
209                 if ("CCSD".equals(sfdu.getControlAuthorityId())) {
210                     sfdus.add(sfdu);
211                     //Read in second SFDU label
212                     input.read(sfduLabel);
213                     sfdus.add(new SFDULabel(sfduLabel));
214                 }
215             } catch (MalformedSFDULabel e) {
216                 //For now we can ignore this error as there is likely not a header.
217             }
218             
219         }
220         
221         return sfdus;
222     }
223 
224     /* (non-Javadoc)
225      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL, gov.nasa.jpl.pds.tools.dict.Dictionary)
226      */
227     public Label parse(URL url, Dictionary dictionary) throws ParseException, IOException {
228         Label label = null;
229 
230         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString()));
231         try {
232 	        //First parse the file and get back the label object
233 	        label = parseLabel(url);
234         } catch (ParseException pe) {
235         	log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
236         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
237         	throw pe;
238         } catch (IOException ioe) {
239         	log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
240         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
241         	throw ioe;
242         }
243         
244         if (label != null && !label.getStatus().equals(Label.SKIP)) {
245 	        log.log(new ToolsLogRecord(Level.INFO, "Starting semantic validation.", url.toString()));
246 	        label = semanticCheck(url, dictionary, label);
247 	        log.log(new ToolsLogRecord(Level.INFO, "Finished semantic validation.", url.toString()));
248         }
249         
250         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
251         
252         return label;
253     }
254     
255     private Label semanticCheck(URL url, Dictionary dictionary, Label label) {
256     	//Check all the statements
257         List statements = label.getStatements();
258         Collections.sort(statements);
259         CountListener listener = new CountListener();
260         for (Iterator i = statements.iterator(); i.hasNext();) {
261             Statement statement = (Statement) i.next();
262             if (statement instanceof AttributeStatement) {
263                 if (!ElementValidator.isValid(dictionary, (AttributeStatement) statement, listener)) {
264                     label.setStatus(FAIL);
265                 }
266             } else if (statement instanceof ObjectStatement) {
267                 if (!ObjectValidator.isValid(dictionary, (ObjectStatement) statement, listener))
268                     label.setStatus(FAIL);
269             } else if (statement instanceof GroupStatement) {
270                 if (!GroupValidator.isValid(dictionary, (GroupStatement) statement, listener))
271                     label.setStatus(FAIL);
272             }
273         }
274         label.incrementErrors(listener.getNumErrors());
275         label.incrementWarnings(listener.getNumWarnings());
276         
277     	return label;
278     }
279 
280     /* (non-Javadoc)
281      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL, gov.nasa.jpl.pds.tools.dict.Dictionary, boolean)
282      */
283     public Label parse(URL file, Dictionary dictionary, boolean dataObjectValidation)  throws ParseException, IOException {
284         return parse(file, dictionary);
285     }
286 
287     /* (non-Javadoc)
288      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#setProperties(java.util.Properties)
289      */
290     public void setProperties(Properties properties) {
291         this.properties.putAll(properties);
292     }
293 
294     /* (non-Javadoc)
295      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getProperties()
296      */
297     public Properties getProperties() {
298         return properties;
299     }
300 
301     /* (non-Javadoc)
302      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getPDSVersion()
303      */
304     public String getPDSVersion() {
305         return "PDS3";
306     }
307 
308     /* (non-Javadoc)
309      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getODLVersion()
310      */
311     public String getODLVersion() {
312         return "2.1";
313     }
314 
315     /* (non-Javadoc)
316      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL)
317      */
318     public Label parsePartial(URL url) throws ParseException, IOException {
319         Label label =  null;
320         
321         try {
322         	label = parsePartial(null, url);
323         } catch (ParseException pe) {
324         	log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
325         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
326         	throw pe;
327         } catch (IOException ioe) {
328         	log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
329         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
330         	throw ioe;
331         }
332         
333         CountListener listener = new CountListener();
334         for (Iterator i = fragmentValidators.iterator(); i.hasNext();) {
335             LabelValidator validator = (LabelValidator) i.next();
336             boolean valid = validator.isValid(label, listener);
337             if (!valid)
338                 label.setStatus(FAIL);
339         }
340         label.incrementErrors(listener.getNumErrors());
341         label.incrementWarnings(listener.getNumWarnings());
342         
343         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
344         
345         return label;
346     }
347     
348     /***
349      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(String,java.net.URL)
350      */
351     public Label parsePartial(String context, URL url) throws ParseException, IOException {
352         Label label = null;
353         
354         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString(), context));
355         log.log(new ToolsLogRecord(ToolsLevel.INFO, "Parsing label fragment", url.toString(), context));
356         
357         //Not all streams can support marking so stream will be open multiple times to look for header
358         //First time to process the SFDUs
359         InputStream sfduCheck = url.openStream();
360         
361         List sfdus = consumeSFDUHeader(sfduCheck);
362         
363         //On the next input stream we will need to skip 20 bytes for every SFDULabel
364         int skip = sfdus.size()*20;
365         //Also add 2 for carriage return line feed if there is a header
366         if (skip != 0)
367             skip += 2;
368         
369         sfduCheck.close();
370         if (sfdus.size() > 0) {
371             log.log(new ToolsLogRecord(Level.WARNING, "Label fragments should not contain SFDU headers."));
372         }
373         
374         InputStream input = url.openStream();
375         input.skip(skip);
376         ODLLexer lexer = new ODLLexer(input);
377         lexer.setFilename(url.toString());
378         lexer.setContext(context);
379         ODLParser parser = new ODLParser(lexer);
380         parser.setFilename(url.toString());
381         parser.setContext(context);
382         
383         if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
384             URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
385             addIncludePath(base);
386             parser.setIncludePaths(includePaths);
387         } else {
388             log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString(), context));
389         }
390         
391         try {
392             label = parser.label();
393             label.setStatus(PASS);
394             label.setStatus(lexer.getStatus());
395             label.incrementErrors(lexer.getNumErrors());
396             label.incrementWarnings(lexer.getNumWarnings());
397             label.setStatus(parser.getStatus());
398             label.incrementErrors(parser.getNumErrors());
399             label.incrementWarnings(parser.getNumWarnings());
400         } catch (ANTLRException ex) {
401             label.setStatus(FAIL);
402             label.incrementErrors();
403             log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString(), context));
404             throw new ParseException(ex.getMessage());
405         }
406         
407         if (label.getStatement("PDS_VERSION_ID") != null) {
408             label.incrementWarnings();
409             log.log(new ToolsLogRecord(Level.WARNING, "Fragment contains PDS_VERSION_ID which should not be present in a label fragment.", url.toString(), context));
410         }
411 
412         log.log(new ToolsLogRecord(ToolsLevel.INFO, "Finished parsing label fragment", url.toString(), context));
413         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "END", url.toString(), context));
414         
415         return label;
416     }
417 
418     /* (non-Javadoc)
419      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, gov.nasa.pds.tools.dict.Dictionary)
420      */
421     public Label parsePartial(URL url, Dictionary dictionary) throws ParseException, IOException {
422     	Label label =  null;
423         
424         try {
425         	label = parsePartial(null, url);
426         } catch (ParseException pe) {
427         	log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
428         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
429         	throw pe;
430         } catch (IOException ioe) {
431         	log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
432         	log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
433         	throw ioe;
434         }
435         
436         CountListener listener = new CountListener();
437         for (Iterator i = fragmentValidators.iterator(); i.hasNext();) {
438             LabelValidator validator = (LabelValidator) i.next();
439             boolean valid = validator.isValid(label, listener);
440             if (!valid)
441                 label.setStatus(FAIL);
442         }
443         label.incrementErrors(listener.getNumErrors());
444         label.incrementWarnings(listener.getNumWarnings());
445         
446     	log.log(new ToolsLogRecord(Level.INFO, "Starting semantic validation.", url.toString()));
447         label = semanticCheck(url, dictionary, label);
448         log.log(new ToolsLogRecord(Level.INFO, "Finished semantic validation.", url.toString()));
449 
450         log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
451     	
452     	return label;
453     }
454 
455     /* (non-Javadoc)
456      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, gov.nasa.pds.tools.dict.Dictionary, boolean)
457      */
458     public Label parsePartial(URL url, Dictionary dictionary, boolean dataObjectValidation) throws ParseException, IOException {
459     	return parsePartial(url, dictionary);
460     }
461     
462     /***
463      * 
464      * @param args
465      * @throws Exception
466      */
467     public static void main(String [] args) throws Exception {
468         Logger logger = Logger.getLogger("");
469         
470         Handler [] handler = logger.getHandlers();
471         for (int i = 0; i < logger.getHandlers().length; i++)
472             logger.removeHandler(handler[i]);
473         
474         StreamHandler stream = new StreamHandler(System.out, new ToolsLogFormatter());
475         logger.addHandler(stream);
476         logger.setLevel(Level.ALL);
477         
478         LabelParserFactory factory = LabelParserFactory.getInstance();
479         LabelParser parser = factory.newLabelParser();
480         Label label = null;
481         URL labelURL = null;
482         URL dictionaryURL = null;
483         URL includePathURL = null;
484         Boolean pointers = null; 
485         Boolean aliasing = null;
486         
487         if (args.length%2 == 0) {
488             for (int i=0; i<args.length; i+=2) {
489                 if (args[i].equals("--label") || args[i].equals("--l"))
490                     labelURL = new URL(args[i+1]);
491                 else if (args[i].equals("--dictionary") || args[i].equals("--d"))
492                     dictionaryURL = new URL(args[i+1]);
493                 else if (args[i].equals("--include") || args[i].equals("--i"))
494                     includePathURL = new URL(args[i+1]);
495                 else if (args[i].equals("--pointers") || args[i].equals("--p"))
496                     pointers = Boolean.valueOf(args[i+1]);
497                 else if (args[i].equals("--aliasing") || args[i].equals("--a"))
498                     aliasing = Boolean.valueOf(args[i+1]);
499                 else {
500                     System.out.println("Invalid flag " + args[i]);
501                     System.exit(1);
502                 }
503             }
504         }
505         
506         if (pointers != null) {
507             parser.getProperties().setProperty("parser.pointers", pointers.toString());
508         }
509         
510         if (includePathURL != null)
511             parser.addIncludePath(includePathURL);
512         
513         Logger logFile = Logger.getLogger(DefaultLabelParser.class.getName());
514         if (dictionaryURL == null) {
515             label = parser.parse(labelURL);
516             System.out.println("Errors: " + label.getNumErrors());
517             System.out.println("Warnings: " + label.getNumWarnings());
518         } else {
519             Dictionary dictionary = DictionaryParser.parse(dictionaryURL, aliasing.booleanValue());
520             logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, dictionary.getStatus(), dictionaryURL.toString()));
521             label = parser.parse(labelURL, dictionary);
522             System.out.println("Errors: " + label.getNumErrors());
523             System.out.println("Warnings: " + label.getNumWarnings());
524         }
525         
526         stream.flush();
527         stream.close();
528     }
529 
530     /* (non-Javadoc)
531      * @see gov.nasa.pds.tools.label.parser.LabelParser#addIncludePath(java.net.URL)
532      */
533 	public void addIncludePath(URL includePath) {
534         if (!includePaths.contains(includePath))
535 		   includePaths.add(includePath);
536 	}
537 
538     /* (non-Javadoc)
539      * @see gov.nasa.pds.tools.label.parser.LabelParser#addValidator(gov.nasa.pds.tools.label.validate.LabelValidator)
540      */
541     public void addLabelValidator(LabelValidator validator) {
542         labelValidators.add(validator);
543     }
544     
545     public void addFragmentValidator(LabelValidator validator) {
546     	fragmentValidators.add(validator);
547     }
548 
549 }