1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package gov.nasa.pds.tools.label.parser;
17
18 import gov.nasa.pds.tools.dict.Dictionary;
19 import gov.nasa.pds.tools.dict.parser.DictionaryParser;
20 import gov.nasa.pds.tools.dict.type.UnsupportedTypeException;
21 import gov.nasa.pds.tools.label.AttributeStatement;
22 import gov.nasa.pds.tools.label.GroupStatement;
23 import gov.nasa.pds.tools.label.Label;
24 import gov.nasa.pds.tools.label.MalformedSFDULabel;
25 import gov.nasa.pds.tools.label.ObjectStatement;
26 import gov.nasa.pds.tools.label.SFDULabel;
27 import gov.nasa.pds.tools.label.Statement;
28 import gov.nasa.pds.tools.label.antlr.ODLLexer;
29 import gov.nasa.pds.tools.label.antlr.ODLParser;
30 import gov.nasa.pds.tools.label.validate.CountListener;
31 import gov.nasa.pds.tools.label.validate.DefinitionNotFoundException;
32 import gov.nasa.pds.tools.label.validate.ElementValidator;
33 import gov.nasa.pds.tools.label.validate.GroupValidator;
34 import gov.nasa.pds.tools.label.validate.LabelValidator;
35 import gov.nasa.pds.tools.label.validate.ObjectValidator;
36 import gov.nasa.pds.tools.label.validate.Status;
37 import gov.nasa.pds.tools.logging.ToolsLevel;
38 import gov.nasa.pds.tools.logging.FullLogFormatter;
39 import gov.nasa.pds.tools.logging.ToolsLogRecord;
40
41 import java.io.BufferedReader;
42 import java.io.IOException;
43 import java.io.InputStream;
44 import java.io.InputStreamReader;
45 import java.net.URL;
46 import java.util.ArrayList;
47 import java.util.Collections;
48 import java.util.Iterator;
49 import java.util.List;
50 import java.util.Properties;
51 import java.util.logging.Handler;
52 import java.util.logging.Level;
53 import java.util.logging.Logger;
54 import java.util.logging.StreamHandler;
55
56 import antlr.ANTLRException;
57
58 /***
59 * Default implementation
60 * @author pramirez
61 * @version $Revision: 3836 $
62 *
63 */
64 public class DefaultLabelParser implements LabelParser, Status {
65 private static Logger log = Logger.getLogger(DefaultLabelParser.class.getName());
66 private Properties properties;
67 private List includePaths;
68 private List labelValidators;
69 private List fragmentValidators;
70
71 public DefaultLabelParser() {
72 properties = new Properties();
73 includePaths = new ArrayList();
74 labelValidators = new ArrayList();
75 fragmentValidators = new ArrayList();
76 }
77
78
79
80
81 public Label parse(URL url) throws ParseException, IOException {
82 Label label = null;
83 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString()));
84 try {
85 label = parseLabel(url);
86 } catch (ParseException pe) {
87 log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
88 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
89 throw pe;
90 } catch (IOException ioe) {
91 log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
92 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
93 throw ioe;
94 }
95
96 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
97
98 return label;
99 }
100
101 private Label parseLabel(URL url) throws ParseException, IOException {
102 Label label = null;
103
104
105
106 InputStream sfduCheck = url.openStream();
107
108 List sfdus = consumeSFDUHeader(sfduCheck);
109 for (Iterator i = sfdus.iterator(); i.hasNext();) {
110 log.log(new ToolsLogRecord(Level.INFO, "Found SFDU Label: " + i.next().toString(), url.toString()));
111 }
112
113
114 int skip = sfdus.size()*20;
115
116 if (skip != 0)
117 skip += 2;
118
119 sfduCheck.close();
120
121 InputStream pdsCheck = url.openStream();
122
123 BufferedReader reader = new BufferedReader(new InputStreamReader(pdsCheck));
124 reader.skip(skip);
125 String version = reader.readLine();
126 String[] line = null;
127 if (version != null) {
128 version = version.trim();
129 line = version.split("=");
130 }
131
132 if (version == null || line == null || line.length != 2) {
133 log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
134 label = new Label();
135 label.setStatus(Label.SKIP);
136 label.incrementWarnings();
137 return label;
138 }
139
140 String name = line[0].trim();
141 String value = line[1].trim();
142
143 if (!"PDS_VERSION_ID".equals(name)) {
144 log.log(new ToolsLogRecord(Level.WARNING, "Not a label. Could not find the PDS_VERSION_ID in the first line.", url.toString()));
145 label = new Label();
146 label.setStatus(Label.SKIP);
147 label.incrementWarnings();
148 return label;
149 }
150
151 pdsCheck.close();
152 log.log(new ToolsLogRecord(Level.INFO, "Parsing label with PDS_VERSION_ID = " + value, url.toString()));
153
154 InputStream input = url.openStream();
155 input.skip(skip);
156 ODLLexer lexer = new ODLLexer(input);
157 lexer.setFilename(url.toString());
158 ODLParser parser = new ODLParser(lexer);
159 parser.setFilename(url.toString());
160 parser.setFollowPointers(Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue());
161
162 if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
163 URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
164 addIncludePath(base);
165 parser.setIncludePaths(includePaths);
166 } else {
167 log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString()));
168 }
169
170 try {
171 label = parser.label();
172 label.setStatus(PASS);
173 label.setStatus(lexer.getStatus());
174 label.incrementErrors(lexer.getNumErrors());
175 label.incrementWarnings(lexer.getNumWarnings());
176 label.setStatus(parser.getStatus());
177 label.incrementErrors(parser.getNumErrors());
178 label.incrementWarnings(parser.getNumWarnings());
179 } catch (ANTLRException ex) {
180 label.setStatus(FAIL);
181 label.incrementErrors();
182 log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString()));
183 throw new ParseException(ex.getMessage());
184 }
185
186 log.log(new ToolsLogRecord(Level.INFO, "Finished parsing", url.toString()));
187
188 CountListener listener = new CountListener();
189 for (Iterator i = labelValidators.iterator(); i.hasNext();) {
190 LabelValidator validator = (LabelValidator) i.next();
191 boolean valid = validator.isValid(label, listener);
192 if (!valid)
193 label.setStatus(FAIL);
194 }
195 label.incrementErrors(listener.getNumErrors());
196 label.incrementWarnings(listener.getNumWarnings());
197
198 return label;
199 }
200
201 private List consumeSFDUHeader(InputStream input) throws IOException {
202 List sfdus = new ArrayList();
203
204 byte[] sfduLabel = new byte[20];
205 int count = input.read(sfduLabel);
206 if (count == 20) {
207 try {
208 SFDULabel sfdu = new SFDULabel(sfduLabel);
209 if ("CCSD".equals(sfdu.getControlAuthorityId())) {
210 sfdus.add(sfdu);
211
212 input.read(sfduLabel);
213 sfdus.add(new SFDULabel(sfduLabel));
214 }
215 } catch (MalformedSFDULabel e) {
216
217 }
218
219 }
220
221 return sfdus;
222 }
223
224
225
226
227 public Label parse(URL url, Dictionary dictionary) throws ParseException, IOException {
228 Label label = null;
229
230 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString()));
231 try {
232
233 label = parseLabel(url);
234 } catch (ParseException pe) {
235 log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
236 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
237 throw pe;
238 } catch (IOException ioe) {
239 log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
240 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
241 throw ioe;
242 }
243
244 if (label != null && !label.getStatus().equals(Label.SKIP)) {
245 log.log(new ToolsLogRecord(Level.INFO, "Starting semantic validation.", url.toString()));
246 label = semanticCheck(url, dictionary, label);
247 log.log(new ToolsLogRecord(Level.INFO, "Finished semantic validation.", url.toString()));
248 }
249
250 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
251
252 return label;
253 }
254
255 private Label semanticCheck(URL url, Dictionary dictionary, Label label) {
256
257 List statements = label.getStatements();
258 Collections.sort(statements);
259 CountListener listener = new CountListener();
260 for (Iterator i = statements.iterator(); i.hasNext();) {
261 Statement statement = (Statement) i.next();
262 if (statement instanceof AttributeStatement) {
263 if (!ElementValidator.isValid(dictionary, (AttributeStatement) statement, listener)) {
264 label.setStatus(FAIL);
265 }
266 } else if (statement instanceof ObjectStatement) {
267 if (!ObjectValidator.isValid(dictionary, (ObjectStatement) statement, listener))
268 label.setStatus(FAIL);
269 } else if (statement instanceof GroupStatement) {
270 if (!GroupValidator.isValid(dictionary, (GroupStatement) statement, listener))
271 label.setStatus(FAIL);
272 }
273 }
274 label.incrementErrors(listener.getNumErrors());
275 label.incrementWarnings(listener.getNumWarnings());
276
277 return label;
278 }
279
280
281
282
283 public Label parse(URL file, Dictionary dictionary, boolean dataObjectValidation) throws ParseException, IOException {
284 return parse(file, dictionary);
285 }
286
287
288
289
290 public void setProperties(Properties properties) {
291 this.properties.putAll(properties);
292 }
293
294
295
296
297 public Properties getProperties() {
298 return properties;
299 }
300
301
302
303
304 public String getPDSVersion() {
305 return "PDS3";
306 }
307
308
309
310
311 public String getODLVersion() {
312 return "2.1";
313 }
314
315
316
317
318 public Label parsePartial(URL url) throws ParseException, IOException {
319 Label label = null;
320
321 try {
322 label = parsePartial(null, url);
323 } catch (ParseException pe) {
324 log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
325 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
326 throw pe;
327 } catch (IOException ioe) {
328 log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
329 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
330 throw ioe;
331 }
332
333 CountListener listener = new CountListener();
334 for (Iterator i = fragmentValidators.iterator(); i.hasNext();) {
335 LabelValidator validator = (LabelValidator) i.next();
336 boolean valid = validator.isValid(label, listener);
337 if (!valid)
338 label.setStatus(FAIL);
339 }
340 label.incrementErrors(listener.getNumErrors());
341 label.incrementWarnings(listener.getNumWarnings());
342
343 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
344
345 return label;
346 }
347
348 /***
349 * @see gov.nasa.pds.tools.label.parser.LabelParser#parsePartial(String,java.net.URL, boolean)
350 */
351 public Label parsePartial(String context, URL url) throws ParseException, IOException {
352 Label label = null;
353
354 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "BEGIN", url.toString(), context));
355 log.log(new ToolsLogRecord(ToolsLevel.INFO, "Parsing label fragment", url.toString(), context));
356
357
358
359 InputStream sfduCheck = url.openStream();
360
361 List sfdus = consumeSFDUHeader(sfduCheck);
362
363
364 int skip = sfdus.size()*20;
365
366 if (skip != 0)
367 skip += 2;
368
369 sfduCheck.close();
370 if (sfdus.size() > 0) {
371 log.log(new ToolsLogRecord(Level.WARNING, "Label fragments should not contain SFDU headers."));
372 }
373
374 InputStream input = url.openStream();
375 input.skip(skip);
376 ODLLexer lexer = new ODLLexer(input);
377 lexer.setFilename(url.toString());
378 lexer.setContext(context);
379 ODLParser parser = new ODLParser(lexer);
380 parser.setFilename(url.toString());
381 parser.setContext(context);
382 parser.setFollowPointers(Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue());
383
384 if (Boolean.valueOf(properties.getProperty("parser.pointers", "true")).booleanValue()) {
385 URL base = new URL(url.toString().substring(0, url.toString().lastIndexOf("/")));
386 addIncludePath(base);
387 parser.setIncludePaths(includePaths);
388 } else {
389 log.log(new ToolsLogRecord(Level.INFO, "Pointers disabled. Pointers will not be followed.", url.toString(), context));
390 }
391
392 try {
393 label = parser.label();
394 label.setStatus(PASS);
395 label.setStatus(lexer.getStatus());
396 label.incrementErrors(lexer.getNumErrors());
397 label.incrementWarnings(lexer.getNumWarnings());
398 label.setStatus(parser.getStatus());
399 label.incrementErrors(parser.getNumErrors());
400 label.incrementWarnings(parser.getNumWarnings());
401 } catch (ANTLRException ex) {
402 label.setStatus(FAIL);
403 label.incrementErrors();
404 log.log(new ToolsLogRecord(Level.SEVERE, ex.getMessage(), url.toString(), context));
405 throw new ParseException(ex.getMessage());
406 }
407
408 if (label.getStatement("PDS_VERSION_ID") != null) {
409 label.incrementWarnings();
410 log.log(new ToolsLogRecord(Level.WARNING, "Fragment contains PDS_VERSION_ID which should not be present in a label fragment.", url.toString(), context));
411 }
412
413 log.log(new ToolsLogRecord(ToolsLevel.INFO, "Finished parsing label fragment", url.toString(), context));
414 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, "END", url.toString(), context));
415
416 return label;
417 }
418
419
420
421
422 public Label parsePartial(URL url, Dictionary dictionary) throws ParseException, IOException {
423 return parsePartial(url, dictionary, false);
424 }
425
426
427
428
429 public Label parsePartial(URL url, Dictionary dictionary, boolean skipVersion) throws ParseException, IOException {
430 Label label = null;
431
432 try {
433 label = parsePartial(null, url);
434 } catch (ParseException pe) {
435 log.log(new ToolsLogRecord(ToolsLevel.SEVERE, pe.getMessage(), url.toString()));
436 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.FAIL, url.toString()));
437 throw pe;
438 } catch (IOException ioe) {
439 log.log(new ToolsLogRecord(ToolsLevel.WARNING, ioe.getMessage(), url.toString()));
440 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, Label.SKIP, url.toString()));
441 throw ioe;
442 }
443
444 CountListener listener = new CountListener();
445 for (Iterator i = fragmentValidators.iterator(); i.hasNext();) {
446 LabelValidator validator = (LabelValidator) i.next();
447 boolean valid = validator.isValid(label, listener);
448 if (!valid)
449 label.setStatus(FAIL);
450 }
451 label.incrementErrors(listener.getNumErrors());
452 label.incrementWarnings(listener.getNumWarnings());
453
454 log.log(new ToolsLogRecord(Level.INFO, "Starting semantic validation.", url.toString()));
455 label = semanticCheck(url, dictionary, label);
456 log.log(new ToolsLogRecord(Level.INFO, "Finished semantic validation.", url.toString()));
457
458 log.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, label.getStatus(), url.toString()));
459
460 return label;
461 }
462
463
464 /***
465 *
466 * @param args
467 * @throws Exception
468 */
469 public static void main(String [] args) throws Exception {
470 Logger logger = Logger.getLogger("");
471
472 Handler [] handler = logger.getHandlers();
473 for (int i = 0; i < logger.getHandlers().length; i++)
474 logger.removeHandler(handler[i]);
475
476 StreamHandler stream = new StreamHandler(System.out, new FullLogFormatter());
477 logger.addHandler(stream);
478 logger.setLevel(Level.ALL);
479
480 LabelParserFactory factory = LabelParserFactory.getInstance();
481 LabelParser parser = factory.newLabelParser();
482 Label label = null;
483 URL labelURL = null;
484 URL dictionaryURL = null;
485 URL includePathURL = null;
486 Boolean pointers = null;
487 Boolean aliasing = new Boolean(true);
488
489 if (args.length%2 == 0) {
490 for (int i=0; i<args.length; i+=2) {
491 if (args[i].equals("--label") || args[i].equals("--l"))
492 labelURL = new URL(args[i+1]);
493 else if (args[i].equals("--dictionary") || args[i].equals("--d"))
494 dictionaryURL = new URL(args[i+1]);
495 else if (args[i].equals("--include") || args[i].equals("--i"))
496 includePathURL = new URL(args[i+1]);
497 else if (args[i].equals("--pointers") || args[i].equals("--p"))
498 pointers = Boolean.valueOf(args[i+1]);
499 else if (args[i].equals("--aliasing") || args[i].equals("--a"))
500 aliasing = Boolean.valueOf(args[i+1]);
501 else {
502 System.out.println("Invalid flag " + args[i]);
503 System.exit(1);
504 }
505 }
506 }
507
508 if (pointers != null) {
509 parser.getProperties().setProperty("parser.pointers", pointers.toString());
510 }
511
512 if (includePathURL != null)
513 parser.addIncludePath(includePathURL);
514
515 Logger logFile = Logger.getLogger(DefaultLabelParser.class.getName());
516 if (dictionaryURL == null) {
517 label = parser.parse(labelURL);
518 System.out.println("Errors: " + label.getNumErrors());
519 System.out.println("Warnings: " + label.getNumWarnings());
520 } else {
521 Dictionary dictionary = DictionaryParser.parse(dictionaryURL, aliasing.booleanValue());
522 logFile.log(new ToolsLogRecord(ToolsLevel.NOTIFICATION, dictionary.getStatus(), dictionaryURL.toString()));
523 label = parser.parse(labelURL, dictionary);
524 System.out.println("Errors: " + label.getNumErrors());
525 System.out.println("Warnings: " + label.getNumWarnings());
526 }
527
528 stream.flush();
529 stream.close();
530 }
531
532
533
534
535 public void addIncludePath(URL includePath) {
536 if (!includePaths.contains(includePath))
537 includePaths.add(includePath);
538 }
539
540
541
542
543 public void addLabelValidator(LabelValidator validator) {
544 labelValidators.add(validator);
545 }
546
547 public void addFragmentValidator(LabelValidator validator) {
548 fragmentValidators.add(validator);
549 }
550
551 }