View Javadoc

1   // Copyright 2006-2007, by the California Institute of Technology.
2   // ALL RIGHTS RESERVED. United States Government Sponsorship acknowledged.
3   // Any commercial use must be negotiated with the Office of Technology Transfer
4   // at the California Institute of Technology.
5   //
6   // This software is subject to U. S. export control laws and regulations
7   // (22 C.F.R. 120-130 and 15 C.F.R. 730-774). To the extent that the software
8   // is subject to U.S. export control laws and regulations, the recipient has
9   // the responsibility to obtain export licenses or other export authority as
10  // may be required before exporting such information to foreign countries or
11  // providing access to foreign nationals.
12  //
13  // $Id: DefaultLabelParser.java 2652 2007-04-30 15:18:41Z mcayanan $ 
14  //
15  
16  package gov.nasa.pds.tools.label.parser;
17  
18  import gov.nasa.pds.tools.dict.Dictionary;
19  import gov.nasa.pds.tools.dict.parser.DictionaryParser;
20  import gov.nasa.pds.tools.dict.type.UnsupportedTypeException;
21  import gov.nasa.pds.tools.label.AttributeStatement;
22  import gov.nasa.pds.tools.label.GroupStatement;
23  import gov.nasa.pds.tools.label.Label;
24  import gov.nasa.pds.tools.label.MalformedSFDULabel;
25  import gov.nasa.pds.tools.label.ObjectStatement;
26  import gov.nasa.pds.tools.label.SFDULabel;
27  import gov.nasa.pds.tools.label.Statement;
28  import gov.nasa.pds.tools.label.antlr.ODLLexer;
29  import gov.nasa.pds.tools.label.antlr.ODLParser;
30  import gov.nasa.pds.tools.label.validate.CountListener;
31  import gov.nasa.pds.tools.label.validate.DefinitionNotFoundException;
32  import gov.nasa.pds.tools.label.validate.ElementValidator;
33  import gov.nasa.pds.tools.label.validate.GroupValidator;
34  import gov.nasa.pds.tools.label.validate.LabelValidator;
35  import gov.nasa.pds.tools.label.validate.ObjectValidator;
36  import gov.nasa.pds.tools.label.validate.Status;
37  import gov.nasa.pds.tools.logging.ToolsLevel;
38  import gov.nasa.pds.tools.logging.ToolsLogFormatter;
39  import gov.nasa.pds.tools.logging.ToolsLogRecord;
40  
41  import java.io.BufferedReader;
42  import java.io.IOException;
43  import java.io.InputStream;
44  import java.io.InputStreamReader;
45  import java.net.URL;
46  import java.util.ArrayList;
47  import java.util.Collections;
48  import java.util.Iterator;
49  import java.util.List;
50  import java.util.Properties;
51  import java.util.logging.Handler;
52  import java.util.logging.Level;
53  import java.util.logging.Logger;
54  import java.util.logging.StreamHandler;
55  
56  import antlr.ANTLRException;
57  
58  /***
59   * Default implementation
60   * @author pramirez
61   * @version $Revision: 2652 $
62   * 
63   */
64  public class DefaultLabelParser implements LabelParser, Status {
65      private static Logger log = Logger.getLogger(DefaultLabelParser.class.getName());
66      private Properties properties;
67      private List includePaths;
68      private List validators;
69      
70      public DefaultLabelParser() {
71          properties = new Properties();
72          includePaths = new ArrayList();
73          validators = new ArrayList();
74      }
75  
76      /* (non-Javadoc)
77       * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL)
78       */
79      public Label parse(URL url) throws ParseException, IOException {
80          Label label = null;
81          
82          //Not all streams can support marking so stream will be open multiple times to look for header
83          //First time to process the SFDUs
84          InputStream sfduCheck = url.openStream();
85          
86          List sfdus = consumeSFDUHeader(sfduCheck);
87          for (Iterator i = sfdus.iterator(); i.hasNext();) {
88              log.log(new ToolsLogRecord(Level.INFO, "Found SFDU Label: " + i.next().toString(), url.toString()));
89          }
90          
91          //On the next input stream we will need to skip 20 bytes for every SFDULabel
92          int skip = sfdus.size()*20;
93          //Also add 2 for carriage return line feed if there is a header
94          if (skip != 0)
95              skip += 2;
96          
97          sfduCheck.close();
98          
99          InputStream pdsCheck = url.openStream();
100         //Now look for PDS_VERSION_ID to ensure that this is a file we want to validate
101         BufferedReader reader = new BufferedReader(new InputStreamReader(pdsCheck));
102         reader.skip(skip);
103         String version = reader.readLine().trim();
104         String[] line = version.split("=");  
105         
106         if (line.length != 2) {
107             log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
108             throw new ParseException("Not a label. Could not find the PDS_VERSION_ID in the first line.");
109         }
110         
111         String name = line[0].trim();
112         String value = line[1].trim();
113           
114         if (!"PDS_VERSION_ID".equals(name)) {
115             log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
116             throw new ParseException("Not a label. Could not find the PDS_VERSION_ID in the first line.");
117         }
118         
119         pdsCheck.close();
120         InputStream input = url.openStream();
121         input.skip(skip);
122         ODLLexer lexer = new ODLLexer(input);
123         lexer.setFilename(url.toString());
124         ODLParser parser = new ODLParser(lexer);
125         parser.setFilename(url.toString());
126         parser.setFollowPointers(Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue());
127         log.log(new ToolsLogRecord(Level.INFO, "Parsing label with PDS_VERSION_ID = " + value, url.toString()));
128         
129         if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
130             URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
131             addIncludePath(base);
132             parser.setIncludePaths(includePaths);
133         } else {
134             log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString()));
135         }
136         
137         try {
138             label = parser.label();
139             label.setStatus(PASS);
140             label.setStatus(lexer.getStatus());
141             label.incrementErrors(lexer.getNumErrors());
142             label.incrementWarnings(lexer.getNumWarnings());
143             label.setStatus(parser.getStatus());
144             label.incrementErrors(parser.getNumErrors());
145             label.incrementWarnings(parser.getNumWarnings());
146         } catch (ANTLRException ex) {
147             label.setStatus(FAIL);
148             label.incrementErrors();
149             log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString()));
150             throw new ParseException(ex.getMessage());
151         }
152         
153         log.log(new ToolsLogRecord(Level.INFO, "Finished parsing", url.toString()));
154         
155         CountListener listener = new CountListener();
156         for (Iterator i = validators.iterator(); i.hasNext();) {
157             LabelValidator validator = (LabelValidator) i.next();
158             boolean valid = validator.isValid(label, listener);
159             if (!valid)
160                 label.setStatus(FAIL);
161         }
162         label.incrementErrors(listener.getNumErrors());
163         label.incrementWarnings(listener.getNumWarnings());
164 
165         return label;
166     }
167     
168     private List consumeSFDUHeader(InputStream input) throws IOException {
169         List sfdus = new ArrayList();
170         
171         byte[] sfduLabel = new byte[20];
172         int count = input.read(sfduLabel);
173         if (count == 20) {
174             try {
175                 SFDULabel sfdu = new SFDULabel(sfduLabel);
176                 if ("CCSD".equals(sfdu.getControlAuthorityId())) {
177                     sfdus.add(sfdu);
178                     //Read in second SFDU label
179                     input.read(sfduLabel);
180                     sfdus.add(new SFDULabel(sfduLabel));
181                 }
182             } catch (MalformedSFDULabel e) {
183                 //For now we can ignore this error as there is likely not a header.
184             }
185             
186         }
187         
188         return sfdus;
189     }
190 
191     /* (non-Javadoc)
192      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL, gov.nasa.jpl.pds.tools.dict.Dictionary)
193      */
194     public Label parse(URL url, Dictionary dictionary) throws ParseException, IOException {
195         Label label = null;
196         
197         //First parse the file and get back the label object
198         label = parse(url);
199         
200         log.log(new ToolsLogRecord(Level.INFO, "Starting semantic validation.", url.toString()));
201         //Check all the statements
202         List statements = label.getStatements();
203         Collections.sort(statements);
204         CountListener listener = new CountListener();
205         for (Iterator i = statements.iterator(); i.hasNext();) {
206             Statement statement = (Statement) i.next();
207             if (statement instanceof AttributeStatement) {
208                 try {
209                     boolean valid = ElementValidator.isValid(dictionary, (AttributeStatement) statement, listener);
210                     if (!valid) {
211                         label.setStatus(FAIL);
212                     }
213                 } catch (DefinitionNotFoundException dnfe) {
214                     label.setStatus(FAIL);
215                     label.incrementErrors();
216                     log.log(new ToolsLogRecord(Level.SEVERE, dnfe.getMessage(), url.toString(), statement.getLineNumber()));
217                 } catch (UnsupportedTypeException ute) {
218                     label.setStatus(FAIL);
219                     label.incrementErrors();
220                     log.log(new ToolsLogRecord(Level.SEVERE, ute.getMessage(), url.toString(), statement.getLineNumber()));
221                 }
222             } else if (statement instanceof ObjectStatement) {
223                 try {
224                     boolean valid = ObjectValidator.isValid(dictionary, (ObjectStatement) statement, listener);
225                     if (!valid)
226                         label.setStatus(FAIL);
227                 } catch (DefinitionNotFoundException dnfe) {
228                     label.setStatus(FAIL);
229                     label.incrementErrors();
230                     log.log(new ToolsLogRecord(Level.SEVERE, dnfe.getMessage(), url.toString(), statement.getLineNumber()));
231                 }
232             } else if (statement instanceof GroupStatement) {
233                 try {
234                     boolean valid = GroupValidator.isValid(dictionary, (GroupStatement) statement, listener);
235                     if (!valid)
236                         label.setStatus(FAIL);
237                 } catch (DefinitionNotFoundException dnfe) {
238                     label.setStatus(FAIL);
239                     label.incrementErrors();
240                     log.log(new ToolsLogRecord(Level.SEVERE, dnfe.getMessage(), url.toString(), statement.getLineNumber()));
241                 }
242             }
243         }
244         label.incrementErrors(listener.getNumErrors());
245         label.incrementWarnings(listener.getNumWarnings());
246         log.log(new ToolsLogRecord(Level.INFO, "Finished semantic validation.", url.toString()));
247         
248         return label;
249     }
250 
251     /* (non-Javadoc)
252      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL, gov.nasa.jpl.pds.tools.dict.Dictionary, boolean)
253      */
254     public Label parse(URL file, Dictionary dictionary, boolean dataObjectValidation)  throws ParseException, IOException {
255         // TODO Auto-generated method stub
256         return parse(file, dictionary);
257     }
258 
259     /* (non-Javadoc)
260      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#setProperties(java.util.Properties)
261      */
262     public void setProperties(Properties properties) {
263         this.properties.putAll(properties);
264     }
265 
266     /* (non-Javadoc)
267      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getProperties()
268      */
269     public Properties getProperties() {
270         return properties;
271     }
272 
273     /* (non-Javadoc)
274      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getPDSVersion()
275      */
276     public String getPDSVersion() {
277         return "PDS3";
278     }
279 
280     /* (non-Javadoc)
281      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getODLVersion()
282      */
283     public String getODLVersion() {
284         return "2.1";
285     }
286 
287     /* (non-Javadoc)
288      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL)
289      */
290     public Label parsePartial(URL url) throws ParseException, IOException {
291         return parsePartial(null, url);
292     }
293     
294     
295     /***
296      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(String,java.net.URL)
297      */
298     public Label parsePartial(String context, URL url) throws ParseException, IOException {
299         Label label = null;
300         
301         log.log(new ToolsLogRecord(Level.INFO, "Parsing label fragment.", url.toString(), context));
302         
303         //Not all streams can support marking so stream will be open multiple times to look for header
304         //First time to process the SFDUs
305         InputStream sfduCheck = url.openStream();
306         
307         List sfdus = consumeSFDUHeader(sfduCheck);
308         
309         //On the next input stream we will need to skip 20 bytes for every SFDULabel
310         int skip = sfdus.size()*20;
311         //Also add 2 for carriage return line feed if there is a header
312         if (skip != 0)
313             skip += 2;
314         
315         sfduCheck.close();
316         if (sfdus.size() > 0) {
317             log.log(new ToolsLogRecord(Level.WARNING, "Label fragments should not contain SFDU headers."));
318         }
319         
320         InputStream input = url.openStream();
321         input.skip(skip);
322         ODLLexer lexer = new ODLLexer(input);
323         lexer.setFilename(url.toString());
324         lexer.setContext(context);
325         ODLParser parser = new ODLParser(lexer);
326         parser.setFilename(url.toString());
327         parser.setContext(context);
328         
329         if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
330             URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
331             addIncludePath(base);
332             parser.setIncludePaths(includePaths);
333         } else {
334             log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString(), context));
335         }
336         
337         try {
338             label = parser.label();
339             label.setStatus(PASS);
340             label.setStatus(lexer.getStatus());
341             label.incrementErrors(lexer.getNumErrors());
342             label.incrementWarnings(lexer.getNumWarnings());
343             label.setStatus(parser.getStatus());
344             label.incrementErrors(parser.getNumErrors());
345             label.incrementWarnings(parser.getNumWarnings());
346         } catch (ANTLRException ex) {
347             label.setStatus(FAIL);
348             label.incrementErrors();
349             log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString(), context));
350             throw new ParseException(ex.getMessage());
351         }
352         
353         if (label.getStatement("PDS_VERSION_ID") != null) {
354             label.incrementWarnings();
355             log.log(new ToolsLogRecord(Level.WARNING, "Fragment contains PDS_VERSION_ID which should not be present in a label fragment.", url.toString(), context));
356         }
357 
358         log.log(new ToolsLogRecord(Level.INFO, "Finished parsing label fragment.", url.toString(), context));
359         
360         return label;
361     }
362 
363     /* (non-Javadoc)
364      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, gov.nasa.pds.tools.dict.Dictionary)
365      */
366     public Label parsePartial(URL file, Dictionary dictionary) throws ParseException, IOException {
367         // TODO Auto-generated method stub
368         return null;
369     }
370 
371     /* (non-Javadoc)
372      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, gov.nasa.pds.tools.dict.Dictionary, boolean)
373      */
374     public Label parsePartial(URL file, Dictionary dictionary, boolean dataObjectValidation) throws ParseException, IOException {
375         // TODO Auto-generated method stub
376         return null;
377     }
378     
379     /***
380      * 
381      * @param args
382      * @throws Exception
383      */
384     public static void main(String [] args) throws Exception {
385         Logger logger = Logger.getLogger("");
386         
387         Handler [] handler = logger.getHandlers();
388         for (int i = 0; i < logger.getHandlers().length; i++)
389             logger.removeHandler(handler[i]);
390         
391         StreamHandler stream = new StreamHandler(System.out, new ToolsLogFormatter());
392         logger.addHandler(stream);
393         logger.setLevel(Level.ALL);
394         
395         LabelParserFactory factory = LabelParserFactory.getInstance();
396         LabelParser parser = factory.newLabelParser();
397         Label label = null;
398         URL labelURL = null;
399         URL dictionaryURL = null;
400         URL includePathURL = null;
401         Boolean pointers = null; 
402         Boolean aliasing = null;
403         
404         if (args.length%2 == 0) {
405             for (int i=0; i<args.length; i+=2) {
406                 if (args[i].equals("--label") || args[i].equals("--l"))
407                     labelURL = new URL(args[i+1]);
408                 else if (args[i].equals("--dictionary") || args[i].equals("--d"))
409                     dictionaryURL = new URL(args[i+1]);
410                 else if (args[i].equals("--include") || args[i].equals("--i"))
411                     includePathURL = new URL(args[i+1]);
412                 else if (args[i].equals("--pointers") || args[i].equals("--p"))
413                     pointers = Boolean.valueOf(args[i+1]);
414                 else if (args[i].equals("--aliasing") || args[i].equals("--a"))
415                     aliasing = Boolean.valueOf(args[i+1]);
416                 else {
417                     System.out.println("Invalid flag " + args[i]);
418                     System.exit(1);
419                 }
420             }
421         }
422         
423         if (pointers != null) {
424             parser.getProperties().setProperty("parser.pointers", pointers.toString());
425         }
426         
427         if (includePathURL != null)
428             parser.addIncludePath(includePathURL);
429         
430         Logger logFile = Logger.getLogger(DefaultLabelParser.class.getName());
431         if (dictionaryURL == null) {
432             label = parser.parse(labelURL);
433             System.out.println("Errors: " + label.getNumErrors());
434             System.out.println("Warnings: " + label.getNumWarnings());
435             logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), labelURL.toString()));
436         } else {
437             Dictionary dictionary = DictionaryParser.parse(dictionaryURL, aliasing.booleanValue());
438             label = parser.parse(labelURL, dictionary);
439             System.out.println("Errors: " + label.getNumErrors());
440             System.out.println("Warnings: " + label.getNumWarnings());
441             logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), labelURL.toString()));
442             logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, dictionary.getStatus(), dictionaryURL.toString()));
443         }
444         
445         stream.flush();
446         stream.close();
447     }
448 
449     /* (non-Javadoc)
450      * @see gov.nasa.pds.tools.label.parser.LabelParser#addIncludePath(java.net.URL)
451      */
452 	public void addIncludePath(URL includePath) {
453         if (!includePaths.contains(includePath))
454 		   includePaths.add(includePath);
455 	}
456 
457     /* (non-Javadoc)
458      * @see gov.nasa.pds.tools.label.parser.LabelParser#addValidator(gov.nasa.pds.tools.label.validate.LabelValidator)
459      */
460     public void addValidator(LabelValidator validator) {
461         validators.add(validator);
462     }
463 
464 }