View Javadoc

1   // Copyright 2006-2007, by the California Institute of Technology.
2   // ALL RIGHTS RESERVED. United States Government Sponsorship acknowledged.
3   // Any commercial use must be negotiated with the Office of Technology Transfer
4   // at the California Institute of Technology.
5   //
6   // This software is subject to U. S. export control laws and regulations
7   // (22 C.F.R. 120-130 and 15 C.F.R. 730-774). To the extent that the software
8   // is subject to U.S. export control laws and regulations, the recipient has
9   // the responsibility to obtain export licenses or other export authority as
10  // may be required before exporting such information to foreign countries or
11  // providing access to foreign nationals.
12  //
13  // $Id: DefaultLabelParser.java 2652 2007-04-30 15:18:41Z mcayanan $ 
14  //
15  
16  package gov.nasa.pds.tools.label.parser;
17  
18  import gov.nasa.pds.tools.dict.Dictionary;
19  import gov.nasa.pds.tools.dict.parser.DictionaryParser;
20  import gov.nasa.pds.tools.dict.type.UnsupportedTypeException;
21  import gov.nasa.pds.tools.label.AttributeStatement;
22  import gov.nasa.pds.tools.label.GroupStatement;
23  import gov.nasa.pds.tools.label.Label;
24  import gov.nasa.pds.tools.label.MalformedSFDULabel;
25  import gov.nasa.pds.tools.label.ObjectStatement;
26  import gov.nasa.pds.tools.label.SFDULabel;
27  import gov.nasa.pds.tools.label.Statement;
28  import gov.nasa.pds.tools.label.antlr.ODLLexer;
29  import gov.nasa.pds.tools.label.antlr.ODLParser;
30  import gov.nasa.pds.tools.label.validate.DefinitionNotFoundException;
31  import gov.nasa.pds.tools.label.validate.ElementValidator;
32  import gov.nasa.pds.tools.label.validate.GroupValidator;
33  import gov.nasa.pds.tools.label.validate.LabelValidator;
34  import gov.nasa.pds.tools.label.validate.ObjectValidator;
35  import gov.nasa.pds.tools.label.validate.Status;
36  import gov.nasa.pds.tools.logging.ToolsLevel;
37  import gov.nasa.pds.tools.logging.ToolsLogFormatter;
38  import gov.nasa.pds.tools.logging.ToolsLogRecord;
39  
40  import java.io.BufferedReader;
41  import java.io.IOException;
42  import java.io.InputStream;
43  import java.io.InputStreamReader;
44  import java.net.URL;
45  import java.util.ArrayList;
46  import java.util.Collections;
47  import java.util.Iterator;
48  import java.util.List;
49  import java.util.Properties;
50  import java.util.logging.Handler;
51  import java.util.logging.Level;
52  import java.util.logging.Logger;
53  import java.util.logging.StreamHandler;
54  
55  import antlr.ANTLRException;
56  
57  /***
58   * Default implementation
59   * @author pramirez
60   * @version $Revision: 2652 $
61   * 
62   */
63  public class DefaultLabelParser implements LabelParser, Status {
64      private static Logger log = Logger.getLogger(DefaultLabelParser.class.getName());
65      private Properties properties;
66      private List includePaths;
67      private List validators;
68      
69      public DefaultLabelParser() {
70          properties = new Properties();
71          includePaths = new ArrayList();
72          validators = new ArrayList();
73      }
74  
75      /* (non-Javadoc)
76       * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL)
77       */
78      public Label parse(URL url) throws ParseException, IOException {
79          Label label = null;
80          
81          //Not all streams can support marking so stream will be open multiple times to look for header
82          //First time to process the SFDUs
83          InputStream sfduCheck = url.openStream();
84          
85          List sfdus = consumeSFDUHeader(sfduCheck);
86          for (Iterator i = sfdus.iterator(); i.hasNext();) {
87              log.log(new ToolsLogRecord(Level.INFO, "Found SFDU Label: " + i.next().toString(), url.toString()));
88          }
89          
90          //On the next input stream we will need to skip 20 bytes for every SFDULabel
91          int skip = sfdus.size()*20;
92          //Also add 2 for carriage return line feed if there is a header
93          if (skip != 0)
94              skip += 2;
95          
96          sfduCheck.close();
97          
98          InputStream pdsCheck = url.openStream();
99          //Now look for PDS_VERSION_ID to ensure that this is a file we want to validate
100         BufferedReader reader = new BufferedReader(new InputStreamReader(pdsCheck));
101         reader.skip(skip);
102         String version = reader.readLine().trim();
103         String[] line = version.split("=");  
104         
105         if (line.length != 2) {
106             log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
107             throw new ParseException("Not a label. Could not find the PDS_VERSION_ID in the first line.");
108         }
109         
110         String name = line[0].trim();
111         String value = line[1].trim();
112           
113         if (!"PDS_VERSION_ID".equals(name)) {
114             log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
115             throw new ParseException("Not a label. Could not find the PDS_VERSION_ID in the first line.");
116         }
117         
118         pdsCheck.close();
119         InputStream input = url.openStream();
120         input.skip(skip);
121         ODLLexer lexer = new ODLLexer(input);
122         lexer.setFilename(url.toString());
123         ODLParser parser = new ODLParser(lexer);
124         parser.setFilename(url.toString());
125         parser.setFollowPointers(Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue());
126         log.log(new ToolsLogRecord(Level.INFO, "Parsing label with PDS_VERSION_ID = " + value, url.toString()));
127         
128         if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
129             URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
130             addIncludePath(base);
131             parser.setIncludePaths(includePaths);
132         } else {
133             log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString()));
134         }
135         
136         try {
137             label = parser.label();
138             label.setStatus(PASS);
139             label.setStatus(lexer.getStatus());
140             label.setStatus(parser.getStatus());
141         } catch (ANTLRException ex) {
142             label.setStatus(FAIL);
143             log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString()));
144             throw new ParseException(ex.getMessage());
145         }
146         
147         log.log(new ToolsLogRecord(Level.INFO, "Finished parsing", url.toString()));
148         
149         for (Iterator i = validators.iterator(); i.hasNext();) {
150             LabelValidator validator = (LabelValidator) i.next();
151             boolean valid = validator.isValid(label);
152             if (!valid)
153                 label.setStatus(FAIL);
154         }
155 
156         return label;
157     }
158     
159     private List consumeSFDUHeader(InputStream input) throws IOException {
160         List sfdus = new ArrayList();
161         
162         byte[] sfduLabel = new byte[20];
163         int count = input.read(sfduLabel);
164         if (count == 20) {
165             try {
166                 SFDULabel sfdu = new SFDULabel(sfduLabel);
167                 if ("CCSD".equals(sfdu.getControlAuthorityId())) {
168                     sfdus.add(sfdu);
169                     //Read in second SFDU label
170                     input.read(sfduLabel);
171                     sfdus.add(new SFDULabel(sfduLabel));
172                 }
173             } catch (MalformedSFDULabel e) {
174                 //For now we can ignore this error as there is likely not a header.
175             }
176             
177         }
178         
179         return sfdus;
180     }
181 
182     /* (non-Javadoc)
183      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL, gov.nasa.jpl.pds.tools.dict.Dictionary)
184      */
185     public Label parse(URL url, Dictionary dictionary) throws ParseException, IOException {
186         Label label = null;
187         
188         //First parse the file and get back the label object
189         label = parse(url);
190         
191         log.log(new ToolsLogRecord(Level.INFO, "Starting semantic validation.", url.toString()));
192         //Check all the statements
193         List statements = label.getStatements();
194         Collections.sort(statements);
195         for (Iterator i = statements.iterator(); i.hasNext();) {
196             Statement statement = (Statement) i.next();
197             if (statement instanceof AttributeStatement) {
198                 try {
199                     boolean valid = ElementValidator.isValid(dictionary, (AttributeStatement) statement);
200                     if (!valid)
201                         label.setStatus(FAIL);
202                 } catch (DefinitionNotFoundException dnfe) {
203                     label.setStatus(FAIL);
204                     log.log(new ToolsLogRecord(Level.SEVERE, dnfe.getMessage(), url.toString(), statement.getLineNumber()));
205                 } catch (UnsupportedTypeException ute) {
206                     label.setStatus(FAIL);
207                     log.log(new ToolsLogRecord(Level.SEVERE, ute.getMessage(), url.toString(), statement.getLineNumber()));
208                 }
209             } else if (statement instanceof ObjectStatement) {
210                 try {
211                     boolean valid = ObjectValidator.isValid(dictionary, (ObjectStatement) statement);
212                     if (!valid)
213                         label.setStatus(FAIL);
214                 } catch (DefinitionNotFoundException dnfe) {
215                     label.setStatus(FAIL);
216                     log.log(new ToolsLogRecord(Level.SEVERE, dnfe.getMessage(), url.toString(), statement.getLineNumber()));
217                 }
218             } else if (statement instanceof GroupStatement) {
219                 try {
220                     boolean valid = GroupValidator.isValid(dictionary, (GroupStatement) statement);
221                     if (!valid)
222                         label.setStatus(FAIL);
223                 } catch (DefinitionNotFoundException dnfe) {
224                     label.setStatus(FAIL);
225                     log.log(new ToolsLogRecord(Level.SEVERE, dnfe.getMessage(), url.toString(), statement.getLineNumber()));
226                 }
227             }
228         }
229         
230         log.log(new ToolsLogRecord(Level.INFO, "Finished semantic validation.", url.toString()));
231         
232         return label;
233     }
234 
235     /* (non-Javadoc)
236      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#parse(java.net.URL, gov.nasa.jpl.pds.tools.dict.Dictionary, boolean)
237      */
238     public Label parse(URL file, Dictionary dictionary, boolean dataObjectValidation)  throws ParseException, IOException {
239         // TODO Auto-generated method stub
240         return parse(file, dictionary);
241     }
242 
243     /* (non-Javadoc)
244      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#setProperties(java.util.Properties)
245      */
246     public void setProperties(Properties properties) {
247         this.properties.putAll(properties);
248     }
249 
250     /* (non-Javadoc)
251      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getProperties()
252      */
253     public Properties getProperties() {
254         return properties;
255     }
256 
257     /* (non-Javadoc)
258      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getPDSVersion()
259      */
260     public String getPDSVersion() {
261         return "PDS3";
262     }
263 
264     /* (non-Javadoc)
265      * @see gov.nasa.jpl.pds.tools.label.parser.LabelParser#getODLVersion()
266      */
267     public String getODLVersion() {
268         return "2.1";
269     }
270 
271     /* (non-Javadoc)
272      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL)
273      */
274     public Label parsePartial(URL url) throws ParseException, IOException {
275         return parsePartial(null, url);
276     }
277     
278     /***
279      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(String,java.net.URL)
280      */
281     public Label parsePartial(String context, URL url) throws ParseException, IOException {
282         Label label = null;
283         
284         log.log(new ToolsLogRecord(Level.INFO, "Parsing label fragment.", url.toString(), context));
285         
286         //Not all streams can support marking so stream will be open multiple times to look for header
287         //First time to process the SFDUs
288         InputStream sfduCheck = url.openStream();
289         
290         List sfdus = consumeSFDUHeader(sfduCheck);
291         
292         //On the next input stream we will need to skip 20 bytes for every SFDULabel
293         int skip = sfdus.size()*20;
294         //Also add 2 for carriage return line feed if there is a header
295         if (skip != 0)
296             skip += 2;
297         
298         sfduCheck.close();
299         if (sfdus.size() > 0) {
300             log.log(new ToolsLogRecord(Level.WARNING, "Label fragments should not contain SFDU headers."));
301         }
302         
303         InputStream input = url.openStream();
304         input.skip(skip);
305         ODLLexer lexer = new ODLLexer(input);
306         lexer.setFilename(url.toString());
307         lexer.setContext(context);
308         ODLParser parser = new ODLParser(lexer);
309         parser.setFilename(url.toString());
310         parser.setContext(context);
311         
312         if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
313             URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
314             addIncludePath(base);
315             parser.setIncludePaths(includePaths);
316         } else {
317             log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString(), context));
318         }
319         
320         try {
321             label = parser.label();
322             label.setStatus(PASS);
323             label.setStatus(lexer.getStatus());
324             label.setStatus(parser.getStatus());
325         } catch (ANTLRException ex) {
326             label.setStatus(FAIL);
327             log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString(), context));
328             throw new ParseException(ex.getMessage());
329         }
330         
331         if (label.getStatement("PDS_VERSION_ID") != null) {
332             log.log(new ToolsLogRecord(Level.WARNING, "Fragment contains PDS_VERSION_ID which should not be present in a label fragment.", url.toString(), context));
333         }
334 
335         log.log(new ToolsLogRecord(Level.INFO, "Finished parsing label fragment.", url.toString(), context));
336         
337         return label;
338     }
339 
340     /* (non-Javadoc)
341      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, gov.nasa.pds.tools.dict.Dictionary)
342      */
343     public Label parsePartial(URL file, Dictionary dictionary) throws ParseException, IOException {
344         // TODO Auto-generated method stub
345         return null;
346     }
347 
348     /* (non-Javadoc)
349      * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(java.net.URL, gov.nasa.pds.tools.dict.Dictionary, boolean)
350      */
351     public Label parsePartial(URL file, Dictionary dictionary, boolean dataObjectValidation) throws ParseException, IOException {
352         // TODO Auto-generated method stub
353         return null;
354     }
355     
356     /***
357      * 
358      * @param args
359      * @throws Exception
360      */
361     public static void main(String [] args) throws Exception {
362         Logger logger = Logger.getLogger("");
363         
364         Handler [] handler = logger.getHandlers();
365         for (int i = 0; i < logger.getHandlers().length; i++)
366             logger.removeHandler(handler[i]);
367         
368         StreamHandler stream = new StreamHandler(System.out, new ToolsLogFormatter());
369         logger.addHandler(stream);
370         logger.setLevel(Level.ALL);
371         
372         LabelParserFactory factory = LabelParserFactory.getInstance();
373         LabelParser parser = factory.newLabelParser();
374         Label label = null;
375         URL labelURL = null;
376         URL dictionaryURL = null;
377         URL includePathURL = null;
378         Boolean pointers = null; 
379         Boolean aliasing = null;
380         
381         if (args.length%2 == 0) {
382             for (int i=0; i<args.length; i+=2) {
383                 if (args[i].equals("--label") || args[i].equals("--l"))
384                     labelURL = new URL(args[i+1]);
385                 else if (args[i].equals("--dictionary") || args[i].equals("--d"))
386                     dictionaryURL = new URL(args[i+1]);
387                 else if (args[i].equals("--include") || args[i].equals("--i"))
388                     includePathURL = new URL(args[i+1]);
389                 else if (args[i].equals("--pointers") || args[i].equals("--p"))
390                     pointers = Boolean.valueOf(args[i+1]);
391                 else if (args[i].equals("--aliasing") || args[i].equals("--a"))
392                     aliasing = Boolean.valueOf(args[i+1]);
393                 else {
394                     System.out.println("Invalid flag " + args[i]);
395                     System.exit(1);
396                 }
397             }
398         }
399         
400         if (pointers != null) {
401             parser.getProperties().setProperty("parser.pointers", pointers.toString());
402         }
403         
404         if (includePathURL != null)
405             parser.addIncludePath(includePathURL);
406         
407         Logger logFile = Logger.getLogger(DefaultLabelParser.class.getName());
408         if (dictionaryURL == null) {
409             label = parser.parse(labelURL);
410             logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), labelURL.toString()));
411         } else {
412             Dictionary dictionary = DictionaryParser.parse(dictionaryURL, aliasing.booleanValue());
413             label = parser.parse(labelURL, dictionary);
414             logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), labelURL.toString()));
415             logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, dictionary.getStatus(), dictionaryURL.toString()));
416         }
417         
418         stream.flush();
419         stream.close();
420     }
421 
422     /* (non-Javadoc)
423      * @see gov.nasa.pds.tools.label.parser.LabelParser#addIncludePath(java.net.URL)
424      */
425 	public void addIncludePath(URL includePath) {
426         if (!includePaths.contains(includePath))
427 		   includePaths.add(includePath);
428 	}
429 
430     /* (non-Javadoc)
431      * @see gov.nasa.pds.tools.label.parser.LabelParser#addValidator(gov.nasa.pds.tools.label.validate.LabelValidator)
432      */
433     public void addValidator(LabelValidator validator) {
434         validators.add(validator);
435     }
436 
437 }