001/* 002 * Copyright 2008-2019 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2019 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.File; 026import java.io.FileInputStream; 027import java.io.InputStream; 028import java.io.IOException; 029import java.io.OutputStream; 030import java.util.ArrayList; 031import java.util.Iterator; 032import java.util.TreeMap; 033import java.util.LinkedHashMap; 034import java.util.List; 035import java.util.concurrent.atomic.AtomicLong; 036import java.util.zip.GZIPInputStream; 037 038import com.unboundid.ldap.sdk.Entry; 039import com.unboundid.ldap.sdk.LDAPConnection; 040import com.unboundid.ldap.sdk.LDAPException; 041import com.unboundid.ldap.sdk.ResultCode; 042import com.unboundid.ldap.sdk.Version; 043import com.unboundid.ldap.sdk.schema.Schema; 044import com.unboundid.ldap.sdk.schema.EntryValidator; 045import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 046import com.unboundid.ldif.DuplicateValueBehavior; 047import com.unboundid.ldif.LDIFException; 048import com.unboundid.ldif.LDIFReader; 049import com.unboundid.ldif.LDIFReaderEntryTranslator; 050import com.unboundid.ldif.LDIFWriter; 051import com.unboundid.util.Debug; 052import com.unboundid.util.LDAPCommandLineTool; 053import com.unboundid.util.StaticUtils; 054import com.unboundid.util.ThreadSafety; 055import com.unboundid.util.ThreadSafetyLevel; 056import com.unboundid.util.args.ArgumentException; 057import com.unboundid.util.args.ArgumentParser; 058import com.unboundid.util.args.BooleanArgument; 059import com.unboundid.util.args.FileArgument; 060import com.unboundid.util.args.IntegerArgument; 061import com.unboundid.util.args.StringArgument; 062 063 064 065/** 066 * This class provides a simple tool that can be used to validate that the 067 * contents of an LDIF file are valid. This includes ensuring that the contents 068 * can be parsed as valid LDIF, and it can also ensure that the LDIF content 069 * conforms to the server schema. It will obtain the schema by connecting to 070 * the server and retrieving the default schema (i.e., the schema which governs 071 * the root DSE). By default, a thorough set of validation will be performed, 072 * but it is possible to disable certain types of validation. 073 * <BR><BR> 074 * Some of the APIs demonstrated by this example include: 075 * <UL> 076 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 077 * package)</LI> 078 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 079 * package)</LI> 080 * <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI> 081 * <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema} 082 * package)</LI> 083 * </UL> 084 * <BR><BR> 085 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 086 * class (to obtain the information to use to connect to the server to read the 087 * schema), as well as the following additional arguments: 088 * <UL> 089 * <LI>"--schemaDirectory {path}" -- specifies the path to a directory 090 * containing files with schema definitions. If this argument is 091 * provided, then no attempt will be made to communicate with a directory 092 * server.</LI> 093 * <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF 094 * file to be validated.</LI> 095 * <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is 096 * compressed.</LI> 097 * <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file 098 * to be written with information about all entries that failed 099 * validation.</LI> 100 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 101 * concurrent threads to use when processing the LDIF. If this is not 102 * provided, then a default of one thread will be used.</LI> 103 * <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation 104 * process should ignore validation failures due to entries that contain 105 * object classes not defined in the server schema.</LI> 106 * <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process 107 * should ignore validation failures due to entries that contain 108 * attributes not defined in the server schema.</LI> 109 * <LI>"--ignoreMalformedDNs" -- indicates that the validation process should 110 * ignore validation failures due to entries with malformed DNs.</LI> 111 * <LI>"--ignoreMissingRDNValues" -- indicates that the validation process 112 * should ignore validation failures due to entries that contain an RDN 113 * attribute value that is not present in the set of entry 114 * attributes.</LI> 115 * <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation 116 * process should ignore validation failures due to entries that either do 117 * not have a structural object class or that have multiple structural 118 * object classes.</LI> 119 * <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation 120 * process should ignore validation failures due to entries containing 121 * auxiliary classes that are not allowed by a DIT content rule, or 122 * abstract classes that are not subclassed by an auxiliary or structural 123 * class contained in the entry.</LI> 124 * <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process 125 * should ignore validation failures due to entries including attributes 126 * that are not allowed or are explicitly prohibited by a DIT content 127 * rule.</LI> 128 * <LI>"--ignoreMissingAttributes" -- indicates that the validation process 129 * should ignore validation failures due to entries missing required 130 * attributes.</LI> 131 * <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation 132 * process should ignore validation failures due to single-valued 133 * attributes containing multiple values.</LI> 134 * <LI>"--ignoreAttributeSyntax" -- indicates that the validation process 135 * should ignore validation failures due to attribute values which violate 136 * the associated attribute syntax.</LI> 137 * <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation 138 * process should ignore validation failures due to attribute values which 139 * violate the associated attribute syntax, but only for the specified 140 * attribute types.</LI> 141 * <LI>"--ignoreNameForms" -- indicates that the validation process should 142 * ignore validation failures due to name form violations (in which the 143 * entry's RDN does not comply with the associated name form).</LI> 144 * </UL> 145 */ 146@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 147public final class ValidateLDIF 148 extends LDAPCommandLineTool 149 implements LDIFReaderEntryTranslator 150{ 151 /** 152 * The end-of-line character for this platform. 153 */ 154 private static final String EOL = 155 StaticUtils.getSystemProperty("line.separator", "\n"); 156 157 158 159 // The arguments used by this program. 160 private BooleanArgument ignoreDuplicateValues; 161 private BooleanArgument ignoreUndefinedObjectClasses; 162 private BooleanArgument ignoreUndefinedAttributes; 163 private BooleanArgument ignoreMalformedDNs; 164 private BooleanArgument ignoreMissingRDNValues; 165 private BooleanArgument ignoreMissingSuperiorObjectClasses; 166 private BooleanArgument ignoreStructuralObjectClasses; 167 private BooleanArgument ignoreProhibitedObjectClasses; 168 private BooleanArgument ignoreProhibitedAttributes; 169 private BooleanArgument ignoreMissingAttributes; 170 private BooleanArgument ignoreSingleValuedAttributes; 171 private BooleanArgument ignoreAttributeSyntax; 172 private BooleanArgument ignoreNameForms; 173 private BooleanArgument isCompressed; 174 private FileArgument schemaDirectory; 175 private FileArgument ldifFile; 176 private FileArgument rejectFile; 177 private FileArgument encryptionPassphraseFile; 178 private IntegerArgument numThreads; 179 private StringArgument ignoreSyntaxViolationsForAttribute; 180 181 // The counter used to keep track of the number of entries processed. 182 private final AtomicLong entriesProcessed = new AtomicLong(0L); 183 184 // The counter used to keep track of the number of entries that could not be 185 // parsed as valid entries. 186 private final AtomicLong malformedEntries = new AtomicLong(0L); 187 188 // The entry validator that will be used to validate the entries. 189 private EntryValidator entryValidator; 190 191 // The LDIF writer that will be used to write rejected entries. 192 private LDIFWriter rejectWriter; 193 194 195 196 /** 197 * Parse the provided command line arguments and make the appropriate set of 198 * changes. 199 * 200 * @param args The command line arguments provided to this program. 201 */ 202 public static void main(final String[] args) 203 { 204 final ResultCode resultCode = main(args, System.out, System.err); 205 if (resultCode != ResultCode.SUCCESS) 206 { 207 System.exit(resultCode.intValue()); 208 } 209 } 210 211 212 213 /** 214 * Parse the provided command line arguments and make the appropriate set of 215 * changes. 216 * 217 * @param args The command line arguments provided to this program. 218 * @param outStream The output stream to which standard out should be 219 * written. It may be {@code null} if output should be 220 * suppressed. 221 * @param errStream The output stream to which standard error should be 222 * written. It may be {@code null} if error messages 223 * should be suppressed. 224 * 225 * @return A result code indicating whether the processing was successful. 226 */ 227 public static ResultCode main(final String[] args, 228 final OutputStream outStream, 229 final OutputStream errStream) 230 { 231 final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream); 232 return validateLDIF.runTool(args); 233 } 234 235 236 237 /** 238 * Creates a new instance of this tool. 239 * 240 * @param outStream The output stream to which standard out should be 241 * written. It may be {@code null} if output should be 242 * suppressed. 243 * @param errStream The output stream to which standard error should be 244 * written. It may be {@code null} if error messages 245 * should be suppressed. 246 */ 247 public ValidateLDIF(final OutputStream outStream, 248 final OutputStream errStream) 249 { 250 super(outStream, errStream); 251 } 252 253 254 255 /** 256 * Retrieves the name for this tool. 257 * 258 * @return The name for this tool. 259 */ 260 @Override() 261 public String getToolName() 262 { 263 return "validate-ldif"; 264 } 265 266 267 268 /** 269 * Retrieves the description for this tool. 270 * 271 * @return The description for this tool. 272 */ 273 @Override() 274 public String getToolDescription() 275 { 276 return "Validate the contents of an LDIF file " + 277 "against the server schema."; 278 } 279 280 281 282 /** 283 * Retrieves the version string for this tool. 284 * 285 * @return The version string for this tool. 286 */ 287 @Override() 288 public String getToolVersion() 289 { 290 return Version.NUMERIC_VERSION_STRING; 291 } 292 293 294 295 /** 296 * Indicates whether this tool should provide support for an interactive mode, 297 * in which the tool offers a mode in which the arguments can be provided in 298 * a text-driven menu rather than requiring them to be given on the command 299 * line. If interactive mode is supported, it may be invoked using the 300 * "--interactive" argument. Alternately, if interactive mode is supported 301 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 302 * interactive mode may be invoked by simply launching the tool without any 303 * arguments. 304 * 305 * @return {@code true} if this tool supports interactive mode, or 306 * {@code false} if not. 307 */ 308 @Override() 309 public boolean supportsInteractiveMode() 310 { 311 return true; 312 } 313 314 315 316 /** 317 * Indicates whether this tool defaults to launching in interactive mode if 318 * the tool is invoked without any command-line arguments. This will only be 319 * used if {@link #supportsInteractiveMode()} returns {@code true}. 320 * 321 * @return {@code true} if this tool defaults to using interactive mode if 322 * launched without any command-line arguments, or {@code false} if 323 * not. 324 */ 325 @Override() 326 public boolean defaultsToInteractiveMode() 327 { 328 return true; 329 } 330 331 332 333 /** 334 * Indicates whether this tool should provide arguments for redirecting output 335 * to a file. If this method returns {@code true}, then the tool will offer 336 * an "--outputFile" argument that will specify the path to a file to which 337 * all standard output and standard error content will be written, and it will 338 * also offer a "--teeToStandardOut" argument that can only be used if the 339 * "--outputFile" argument is present and will cause all output to be written 340 * to both the specified output file and to standard output. 341 * 342 * @return {@code true} if this tool should provide arguments for redirecting 343 * output to a file, or {@code false} if not. 344 */ 345 @Override() 346 protected boolean supportsOutputFile() 347 { 348 return true; 349 } 350 351 352 353 /** 354 * Indicates whether this tool should default to interactively prompting for 355 * the bind password if a password is required but no argument was provided 356 * to indicate how to get the password. 357 * 358 * @return {@code true} if this tool should default to interactively 359 * prompting for the bind password, or {@code false} if not. 360 */ 361 @Override() 362 protected boolean defaultToPromptForBindPassword() 363 { 364 return true; 365 } 366 367 368 369 /** 370 * Indicates whether this tool supports the use of a properties file for 371 * specifying default values for arguments that aren't specified on the 372 * command line. 373 * 374 * @return {@code true} if this tool supports the use of a properties file 375 * for specifying default values for arguments that aren't specified 376 * on the command line, or {@code false} if not. 377 */ 378 @Override() 379 public boolean supportsPropertiesFile() 380 { 381 return true; 382 } 383 384 385 386 /** 387 * Indicates whether the LDAP-specific arguments should include alternate 388 * versions of all long identifiers that consist of multiple words so that 389 * they are available in both camelCase and dash-separated versions. 390 * 391 * @return {@code true} if this tool should provide multiple versions of 392 * long identifiers for LDAP-specific arguments, or {@code false} if 393 * not. 394 */ 395 @Override() 396 protected boolean includeAlternateLongIdentifiers() 397 { 398 return true; 399 } 400 401 402 403 /** 404 * Indicates whether this tool should provide a command-line argument that 405 * allows for low-level SSL debugging. If this returns {@code true}, then an 406 * "--enableSSLDebugging}" argument will be added that sets the 407 * "javax.net.debug" system property to "all" before attempting any 408 * communication. 409 * 410 * @return {@code true} if this tool should offer an "--enableSSLDebugging" 411 * argument, or {@code false} if not. 412 */ 413 @Override() 414 protected boolean supportsSSLDebugging() 415 { 416 return true; 417 } 418 419 420 421 /** 422 * Adds the arguments used by this program that aren't already provided by the 423 * generic {@code LDAPCommandLineTool} framework. 424 * 425 * @param parser The argument parser to which the arguments should be added. 426 * 427 * @throws ArgumentException If a problem occurs while adding the arguments. 428 */ 429 @Override() 430 public void addNonLDAPArguments(final ArgumentParser parser) 431 throws ArgumentException 432 { 433 String description = "The path to the LDIF file to process. The tool " + 434 "will automatically attempt to detect whether the file is " + 435 "encrypted or compressed."; 436 ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description, 437 true, true, true, false); 438 ldifFile.addLongIdentifier("ldif-file", true); 439 parser.addArgument(ldifFile); 440 441 442 // Add an argument that makes it possible to read a compressed LDIF file. 443 // Note that this argument is no longer needed for dealing with compressed 444 // files, since the tool will automatically detect whether a file is 445 // compressed. However, the argument is still provided for the purpose of 446 // backward compatibility. 447 description = "Indicates that the specified LDIF file is compressed " + 448 "using gzip compression."; 449 isCompressed = new BooleanArgument('c', "isCompressed", description); 450 isCompressed.addLongIdentifier("is-compressed", true); 451 isCompressed.setHidden(true); 452 parser.addArgument(isCompressed); 453 454 455 // Add an argument that indicates that the tool should read the encryption 456 // passphrase from a file. 457 description = "Indicates that the specified LDIF file is encrypted and " + 458 "that the encryption passphrase is contained in the specified " + 459 "file. If the LDIF data is encrypted and this argument is not " + 460 "provided, then the tool will interactively prompt for the " + 461 "encryption passphrase."; 462 encryptionPassphraseFile = new FileArgument(null, 463 "encryptionPassphraseFile", false, 1, null, description, true, true, 464 true, false); 465 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 466 true); 467 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 468 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 469 true); 470 parser.addArgument(encryptionPassphraseFile); 471 472 473 description = "The path to the file to which rejected entries should be " + 474 "written."; 475 rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}", 476 description, false, true, true, false); 477 rejectFile.addLongIdentifier("reject-file", true); 478 parser.addArgument(rejectFile); 479 480 description = "The path to a directory containing one or more LDIF files " + 481 "with the schema information to use. If this is provided, " + 482 "then no LDAP communication will be performed."; 483 schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1, 484 "{path}", description, true, true, false, true); 485 schemaDirectory.addLongIdentifier("schema-directory", true); 486 parser.addArgument(schemaDirectory); 487 488 description = "The number of threads to use when processing the LDIF file."; 489 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 490 description, 1, Integer.MAX_VALUE, 1); 491 numThreads.addLongIdentifier("num-threads", true); 492 parser.addArgument(numThreads); 493 494 description = "Ignore validation failures due to entries containing " + 495 "duplicate values for the same attribute."; 496 ignoreDuplicateValues = 497 new BooleanArgument(null, "ignoreDuplicateValues", description); 498 ignoreDuplicateValues.setArgumentGroupName( 499 "Validation Strictness Arguments"); 500 ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values", true); 501 parser.addArgument(ignoreDuplicateValues); 502 503 description = "Ignore validation failures due to object classes not " + 504 "defined in the schema."; 505 ignoreUndefinedObjectClasses = 506 new BooleanArgument(null, "ignoreUndefinedObjectClasses", description); 507 ignoreUndefinedObjectClasses.setArgumentGroupName( 508 "Validation Strictness Arguments"); 509 ignoreUndefinedObjectClasses.addLongIdentifier( 510 "ignore-undefined-object-classes", true); 511 parser.addArgument(ignoreUndefinedObjectClasses); 512 513 description = "Ignore validation failures due to attributes not defined " + 514 "in the schema."; 515 ignoreUndefinedAttributes = 516 new BooleanArgument(null, "ignoreUndefinedAttributes", description); 517 ignoreUndefinedAttributes.setArgumentGroupName( 518 "Validation Strictness Arguments"); 519 ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes", 520 true); 521 parser.addArgument(ignoreUndefinedAttributes); 522 523 description = "Ignore validation failures due to entries with malformed " + 524 "DNs."; 525 ignoreMalformedDNs = 526 new BooleanArgument(null, "ignoreMalformedDNs", description); 527 ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments"); 528 ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns", true); 529 parser.addArgument(ignoreMalformedDNs); 530 531 description = "Ignore validation failures due to entries with RDN " + 532 "attribute values that are missing from the set of entry " + 533 "attributes."; 534 ignoreMissingRDNValues = 535 new BooleanArgument(null, "ignoreMissingRDNValues", description); 536 ignoreMissingRDNValues.setArgumentGroupName( 537 "Validation Strictness Arguments"); 538 ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values", true); 539 parser.addArgument(ignoreMissingRDNValues); 540 541 description = "Ignore validation failures due to entries without exactly " + 542 "structural object class."; 543 ignoreStructuralObjectClasses = 544 new BooleanArgument(null, "ignoreStructuralObjectClasses", 545 description); 546 ignoreStructuralObjectClasses.setArgumentGroupName( 547 "Validation Strictness Arguments"); 548 ignoreStructuralObjectClasses.addLongIdentifier( 549 "ignore-structural-object-classes", true); 550 parser.addArgument(ignoreStructuralObjectClasses); 551 552 description = "Ignore validation failures due to entries with object " + 553 "classes that are not allowed."; 554 ignoreProhibitedObjectClasses = 555 new BooleanArgument(null, "ignoreProhibitedObjectClasses", 556 description); 557 ignoreProhibitedObjectClasses.setArgumentGroupName( 558 "Validation Strictness Arguments"); 559 ignoreProhibitedObjectClasses.addLongIdentifier( 560 "ignore-prohibited-object-classes", true); 561 parser.addArgument(ignoreProhibitedObjectClasses); 562 563 description = "Ignore validation failures due to entries that are " + 564 "one or more superior object classes."; 565 ignoreMissingSuperiorObjectClasses = 566 new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses", 567 description); 568 ignoreMissingSuperiorObjectClasses.setArgumentGroupName( 569 "Validation Strictness Arguments"); 570 ignoreMissingSuperiorObjectClasses.addLongIdentifier( 571 "ignore-missing-superior-object-classes", true); 572 parser.addArgument(ignoreMissingSuperiorObjectClasses); 573 574 description = "Ignore validation failures due to entries with attributes " + 575 "that are not allowed."; 576 ignoreProhibitedAttributes = 577 new BooleanArgument(null, "ignoreProhibitedAttributes", description); 578 ignoreProhibitedAttributes.setArgumentGroupName( 579 "Validation Strictness Arguments"); 580 ignoreProhibitedAttributes.addLongIdentifier( 581 "ignore-prohibited-attributes", true); 582 parser.addArgument(ignoreProhibitedAttributes); 583 584 description = "Ignore validation failures due to entries missing " + 585 "required attributes."; 586 ignoreMissingAttributes = 587 new BooleanArgument(null, "ignoreMissingAttributes", description); 588 ignoreMissingAttributes.setArgumentGroupName( 589 "Validation Strictness Arguments"); 590 ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes", 591 true); 592 parser.addArgument(ignoreMissingAttributes); 593 594 description = "Ignore validation failures due to entries with multiple " + 595 "values for single-valued attributes."; 596 ignoreSingleValuedAttributes = 597 new BooleanArgument(null, "ignoreSingleValuedAttributes", description); 598 ignoreSingleValuedAttributes.setArgumentGroupName( 599 "Validation Strictness Arguments"); 600 ignoreSingleValuedAttributes.addLongIdentifier( 601 "ignore-single-valued-attributes", true); 602 parser.addArgument(ignoreSingleValuedAttributes); 603 604 description = "Ignore validation failures due to entries with attribute " + 605 "values that violate their associated syntax. If this is " + 606 "provided, then no attribute syntax violations will be " + 607 "flagged. If this is not provided, then all attribute " + 608 "syntax violations will be flagged except for violations " + 609 "in those attributes excluded by the " + 610 "--ignoreSyntaxViolationsForAttribute argument."; 611 ignoreAttributeSyntax = 612 new BooleanArgument(null, "ignoreAttributeSyntax", description); 613 ignoreAttributeSyntax.setArgumentGroupName( 614 "Validation Strictness Arguments"); 615 ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax", true); 616 parser.addArgument(ignoreAttributeSyntax); 617 618 description = "The name or OID of an attribute for which to ignore " + 619 "validation failures due to violations of the associated " + 620 "attribute syntax. This argument can only be used if the " + 621 "--ignoreAttributeSyntax argument is not provided."; 622 ignoreSyntaxViolationsForAttribute = new StringArgument(null, 623 "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description); 624 ignoreSyntaxViolationsForAttribute.setArgumentGroupName( 625 "Validation Strictness Arguments"); 626 ignoreSyntaxViolationsForAttribute.addLongIdentifier( 627 "ignore-syntax-violations-for-attribute", true); 628 parser.addArgument(ignoreSyntaxViolationsForAttribute); 629 630 description = "Ignore validation failures due to entries with RDNs " + 631 "that violate the associated name form definition."; 632 ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description); 633 ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments"); 634 ignoreNameForms.addLongIdentifier("ignore-name-forms", true); 635 parser.addArgument(ignoreNameForms); 636 637 638 // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments 639 // cannot be used together. 640 parser.addExclusiveArgumentSet(ignoreAttributeSyntax, 641 ignoreSyntaxViolationsForAttribute); 642 } 643 644 645 646 /** 647 * Performs the actual processing for this tool. In this case, it gets a 648 * connection to the directory server and uses it to retrieve the server 649 * schema. It then reads the LDIF file and validates each entry accordingly. 650 * 651 * @return The result code for the processing that was performed. 652 */ 653 @Override() 654 public ResultCode doToolProcessing() 655 { 656 // Get the connection to the directory server and use it to read the schema. 657 final Schema schema; 658 if (schemaDirectory.isPresent()) 659 { 660 final File schemaDir = schemaDirectory.getValue(); 661 662 try 663 { 664 final TreeMap<String,File> fileMap = new TreeMap<>(); 665 for (final File f : schemaDir.listFiles()) 666 { 667 final String name = f.getName(); 668 if (f.isFile() && name.endsWith(".ldif")) 669 { 670 fileMap.put(name, f); 671 } 672 } 673 674 if (fileMap.isEmpty()) 675 { 676 err("No LDIF files found in directory " + 677 schemaDir.getAbsolutePath()); 678 return ResultCode.PARAM_ERROR; 679 } 680 681 final ArrayList<File> fileList = new ArrayList<>(fileMap.values()); 682 schema = Schema.getSchema(fileList); 683 } 684 catch (final Exception e) 685 { 686 Debug.debugException(e); 687 err("Unable to read schema from files in directory " + 688 schemaDir.getAbsolutePath() + ": " + 689 StaticUtils.getExceptionMessage(e)); 690 return ResultCode.LOCAL_ERROR; 691 } 692 } 693 else 694 { 695 try 696 { 697 final LDAPConnection connection = getConnection(); 698 schema = connection.getSchema(); 699 connection.close(); 700 } 701 catch (final LDAPException le) 702 { 703 Debug.debugException(le); 704 err("Unable to connect to the directory server and read the schema: ", 705 le.getMessage()); 706 return le.getResultCode(); 707 } 708 } 709 710 711 // Get the encryption passphrase, if it was provided. 712 String encryptionPassphrase = null; 713 if (encryptionPassphraseFile.isPresent()) 714 { 715 try 716 { 717 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 718 encryptionPassphraseFile.getValue()); 719 } 720 catch (final LDAPException e) 721 { 722 Debug.debugException(e); 723 err(e.getMessage()); 724 return e.getResultCode(); 725 } 726 } 727 728 729 // Create the entry validator and initialize its configuration. 730 entryValidator = new EntryValidator(schema); 731 entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent()); 732 entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent()); 733 entryValidator.setCheckEntryMissingRDNValues( 734 !ignoreMissingRDNValues.isPresent()); 735 entryValidator.setCheckMissingAttributes( 736 !ignoreMissingAttributes.isPresent()); 737 entryValidator.setCheckNameForms(!ignoreNameForms.isPresent()); 738 entryValidator.setCheckProhibitedAttributes( 739 !ignoreProhibitedAttributes.isPresent()); 740 entryValidator.setCheckProhibitedObjectClasses( 741 !ignoreProhibitedObjectClasses.isPresent()); 742 entryValidator.setCheckMissingSuperiorObjectClasses( 743 !ignoreMissingSuperiorObjectClasses.isPresent()); 744 entryValidator.setCheckSingleValuedAttributes( 745 !ignoreSingleValuedAttributes.isPresent()); 746 entryValidator.setCheckStructuralObjectClasses( 747 !ignoreStructuralObjectClasses.isPresent()); 748 entryValidator.setCheckUndefinedAttributes( 749 !ignoreUndefinedAttributes.isPresent()); 750 entryValidator.setCheckUndefinedObjectClasses( 751 !ignoreUndefinedObjectClasses.isPresent()); 752 753 if (ignoreSyntaxViolationsForAttribute.isPresent()) 754 { 755 entryValidator.setIgnoreSyntaxViolationAttributeTypes( 756 ignoreSyntaxViolationsForAttribute.getValues()); 757 } 758 759 760 // Create an LDIF reader that can be used to read through the LDIF file. 761 final LDIFReader ldifReader; 762 rejectWriter = null; 763 try 764 { 765 InputStream inputStream = new FileInputStream(ldifFile.getValue()); 766 767 inputStream = ToolUtils.getPossiblyPassphraseEncryptedInputStream( 768 inputStream, encryptionPassphrase, false, 769 "LDIF file '" + ldifFile.getValue().getPath() + 770 "' is encrypted. Please enter the encryption passphrase:", 771 "ERROR: The provided passphrase was incorrect.", 772 getOut(), getErr()).getFirst(); 773 774 if (isCompressed.isPresent()) 775 { 776 inputStream = new GZIPInputStream(inputStream); 777 } 778 else 779 { 780 inputStream = 781 ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream); 782 } 783 784 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this); 785 } 786 catch (final Exception e) 787 { 788 Debug.debugException(e); 789 err("Unable to open the LDIF reader: ", 790 StaticUtils.getExceptionMessage(e)); 791 return ResultCode.LOCAL_ERROR; 792 } 793 794 ldifReader.setSchema(schema); 795 if (ignoreDuplicateValues.isPresent()) 796 { 797 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP); 798 } 799 else 800 { 801 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT); 802 } 803 804 try 805 { 806 // Create an LDIF writer that can be used to write information about 807 // rejected entries. 808 try 809 { 810 if (rejectFile.isPresent()) 811 { 812 rejectWriter = new LDIFWriter(rejectFile.getValue()); 813 } 814 } 815 catch (final Exception e) 816 { 817 Debug.debugException(e); 818 err("Unable to create the reject writer: ", 819 StaticUtils.getExceptionMessage(e)); 820 return ResultCode.LOCAL_ERROR; 821 } 822 823 ResultCode resultCode = ResultCode.SUCCESS; 824 while (true) 825 { 826 try 827 { 828 final Entry e = ldifReader.readEntry(); 829 if (e == null) 830 { 831 // Because we're performing parallel processing and returning null 832 // from the translate method, LDIFReader.readEntry() should never 833 // return a non-null value. However, it can throw an LDIFException 834 // if it encounters an invalid entry, or an IOException if there's 835 // a problem reading from the file, so we should still iterate 836 // through all of the entries to catch and report on those problems. 837 break; 838 } 839 } 840 catch (final LDIFException le) 841 { 842 Debug.debugException(le); 843 malformedEntries.incrementAndGet(); 844 845 if (resultCode == ResultCode.SUCCESS) 846 { 847 resultCode = ResultCode.DECODING_ERROR; 848 } 849 850 if (rejectWriter != null) 851 { 852 try 853 { 854 rejectWriter.writeComment( 855 "Unable to parse an entry read from LDIF:", false, false); 856 if (le.mayContinueReading()) 857 { 858 rejectWriter.writeComment( 859 StaticUtils.getExceptionMessage(le), false, true); 860 } 861 else 862 { 863 rejectWriter.writeComment( 864 StaticUtils.getExceptionMessage(le), false, 865 false); 866 rejectWriter.writeComment("Unable to continue LDIF processing.", 867 false, true); 868 err("Aborting LDIF processing: ", 869 StaticUtils.getExceptionMessage(le)); 870 return ResultCode.LOCAL_ERROR; 871 } 872 } 873 catch (final IOException ioe) 874 { 875 Debug.debugException(ioe); 876 err("Unable to write to the reject file:", 877 StaticUtils.getExceptionMessage(ioe)); 878 err("LDIF parse failure that triggered the rejection: ", 879 StaticUtils.getExceptionMessage(le)); 880 return ResultCode.LOCAL_ERROR; 881 } 882 } 883 } 884 catch (final IOException ioe) 885 { 886 Debug.debugException(ioe); 887 888 if (rejectWriter != null) 889 { 890 try 891 { 892 rejectWriter.writeComment("I/O error reading from LDIF:", false, 893 false); 894 rejectWriter.writeComment(StaticUtils.getExceptionMessage(ioe), 895 false, true); 896 return ResultCode.LOCAL_ERROR; 897 } 898 catch (final Exception ex) 899 { 900 Debug.debugException(ex); 901 err("I/O error reading from LDIF:", 902 StaticUtils.getExceptionMessage(ioe)); 903 return ResultCode.LOCAL_ERROR; 904 } 905 } 906 } 907 } 908 909 if (malformedEntries.get() > 0) 910 { 911 out(malformedEntries.get() + " entries were malformed and could not " + 912 "be read from the LDIF file."); 913 } 914 915 if (entryValidator.getInvalidEntries() > 0) 916 { 917 if (resultCode == ResultCode.SUCCESS) 918 { 919 resultCode = ResultCode.OBJECT_CLASS_VIOLATION; 920 } 921 922 for (final String s : entryValidator.getInvalidEntrySummary(true)) 923 { 924 out(s); 925 } 926 } 927 else 928 { 929 if (malformedEntries.get() == 0) 930 { 931 out("No errors were encountered."); 932 } 933 } 934 935 return resultCode; 936 } 937 finally 938 { 939 try 940 { 941 ldifReader.close(); 942 } 943 catch (final Exception e) 944 { 945 Debug.debugException(e); 946 } 947 948 try 949 { 950 if (rejectWriter != null) 951 { 952 rejectWriter.close(); 953 } 954 } 955 catch (final Exception e) 956 { 957 Debug.debugException(e); 958 } 959 } 960 } 961 962 963 964 /** 965 * Examines the provided entry to determine whether it conforms to the 966 * server schema. 967 * 968 * @param entry The entry to be examined. 969 * @param firstLineNumber The line number of the LDIF source on which the 970 * provided entry begins. 971 * 972 * @return The updated entry. This method will always return {@code null} 973 * because all of the real processing needed for the entry is 974 * performed in this method and the entry isn't needed any more 975 * after this method is done. 976 */ 977 @Override() 978 public Entry translate(final Entry entry, final long firstLineNumber) 979 { 980 final ArrayList<String> invalidReasons = new ArrayList<>(5); 981 if (! entryValidator.entryIsValid(entry, invalidReasons)) 982 { 983 if (rejectWriter != null) 984 { 985 synchronized (this) 986 { 987 try 988 { 989 rejectWriter.writeEntry(entry, listToString(invalidReasons)); 990 } 991 catch (final IOException ioe) 992 { 993 Debug.debugException(ioe); 994 } 995 } 996 } 997 } 998 999 final long numEntries = entriesProcessed.incrementAndGet(); 1000 if ((numEntries % 1000L) == 0L) 1001 { 1002 out("Processed ", numEntries, " entries."); 1003 } 1004 1005 return null; 1006 } 1007 1008 1009 1010 /** 1011 * Converts the provided list of strings into a single string. It will 1012 * contain line breaks after all but the last element. 1013 * 1014 * @param l The list of strings to convert to a single string. 1015 * 1016 * @return The string from the provided list, or {@code null} if the provided 1017 * list is empty or {@code null}. 1018 */ 1019 private static String listToString(final List<String> l) 1020 { 1021 if ((l == null) || (l.isEmpty())) 1022 { 1023 return null; 1024 } 1025 1026 final StringBuilder buffer = new StringBuilder(); 1027 final Iterator<String> iterator = l.iterator(); 1028 while (iterator.hasNext()) 1029 { 1030 buffer.append(iterator.next()); 1031 if (iterator.hasNext()) 1032 { 1033 buffer.append(EOL); 1034 } 1035 } 1036 1037 return buffer.toString(); 1038 } 1039 1040 1041 1042 /** 1043 * {@inheritDoc} 1044 */ 1045 @Override() 1046 public LinkedHashMap<String[],String> getExampleUsages() 1047 { 1048 final LinkedHashMap<String[],String> examples = 1049 new LinkedHashMap<>(StaticUtils.computeMapCapacity(2)); 1050 1051 String[] args = 1052 { 1053 "--hostname", "server.example.com", 1054 "--port", "389", 1055 "--ldifFile", "data.ldif", 1056 "--rejectFile", "rejects.ldif", 1057 "--numThreads", "4" 1058 }; 1059 String description = 1060 "Validate the contents of the 'data.ldif' file using the schema " + 1061 "defined in the specified directory server using four concurrent " + 1062 "threads. All types of validation will be performed, and " + 1063 "information about any errors will be written to the 'rejects.ldif' " + 1064 "file."; 1065 examples.put(args, description); 1066 1067 1068 args = new String[] 1069 { 1070 "--schemaDirectory", "/ds/config/schema", 1071 "--ldifFile", "data.ldif", 1072 "--rejectFile", "rejects.ldif", 1073 "--ignoreStructuralObjectClasses", 1074 "--ignoreAttributeSyntax" 1075 }; 1076 description = 1077 "Validate the contents of the 'data.ldif' file using the schema " + 1078 "defined in LDIF files contained in the /ds/config/schema directory " + 1079 "using a single thread. Any errors resulting from entries that do " + 1080 "not have exactly one structural object class or from values which " + 1081 "violate the syntax for their associated attribute types will be " + 1082 "ignored. Information about any other failures will be written to " + 1083 "the 'rejects.ldif' file."; 1084 examples.put(args, description); 1085 1086 return examples; 1087 } 1088 1089 1090 1091 /** 1092 * @return EntryValidator 1093 * 1094 * Returns the EntryValidator 1095 */ 1096 public EntryValidator getEntryValidator() 1097 { 1098 return entryValidator; 1099 } 1100}