001/*
002 * Copyright 2008-2020 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2008-2020 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2008-2020 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldap.sdk.examples;
037
038
039
040import java.io.File;
041import java.io.FileInputStream;
042import java.io.InputStream;
043import java.io.IOException;
044import java.io.OutputStream;
045import java.util.ArrayList;
046import java.util.Iterator;
047import java.util.TreeMap;
048import java.util.LinkedHashMap;
049import java.util.List;
050import java.util.concurrent.atomic.AtomicLong;
051import java.util.zip.GZIPInputStream;
052
053import com.unboundid.ldap.sdk.Entry;
054import com.unboundid.ldap.sdk.LDAPConnection;
055import com.unboundid.ldap.sdk.LDAPException;
056import com.unboundid.ldap.sdk.ResultCode;
057import com.unboundid.ldap.sdk.Version;
058import com.unboundid.ldap.sdk.schema.Schema;
059import com.unboundid.ldap.sdk.schema.EntryValidator;
060import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
061import com.unboundid.ldif.DuplicateValueBehavior;
062import com.unboundid.ldif.LDIFException;
063import com.unboundid.ldif.LDIFReader;
064import com.unboundid.ldif.LDIFReaderEntryTranslator;
065import com.unboundid.ldif.LDIFWriter;
066import com.unboundid.util.Debug;
067import com.unboundid.util.LDAPCommandLineTool;
068import com.unboundid.util.StaticUtils;
069import com.unboundid.util.ThreadSafety;
070import com.unboundid.util.ThreadSafetyLevel;
071import com.unboundid.util.args.ArgumentException;
072import com.unboundid.util.args.ArgumentParser;
073import com.unboundid.util.args.BooleanArgument;
074import com.unboundid.util.args.FileArgument;
075import com.unboundid.util.args.IntegerArgument;
076import com.unboundid.util.args.StringArgument;
077
078
079
080/**
081 * This class provides a simple tool that can be used to validate that the
082 * contents of an LDIF file are valid.  This includes ensuring that the contents
083 * can be parsed as valid LDIF, and it can also ensure that the LDIF content
084 * conforms to the server schema.  It will obtain the schema by connecting to
085 * the server and retrieving the default schema (i.e., the schema which governs
086 * the root DSE).  By default, a thorough set of validation will be performed,
087 * but it is possible to disable certain types of validation.
088 * <BR><BR>
089 * Some of the APIs demonstrated by this example include:
090 * <UL>
091 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
092 *       package)</LI>
093 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
094 *       package)</LI>
095 *   <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI>
096 *   <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema}
097 *       package)</LI>
098 * </UL>
099 * <BR><BR>
100 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
101 * class (to obtain the information to use to connect to the server to read the
102 * schema), as well as the following additional arguments:
103 * <UL>
104 *   <LI>"--schemaDirectory {path}" -- specifies the path to a directory
105 *       containing files with schema definitions.  If this argument is
106 *       provided, then no attempt will be made to communicate with a directory
107 *       server.</LI>
108 *   <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF
109 *       file to be validated.</LI>
110 *   <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is
111 *       compressed.</LI>
112 *   <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file
113 *       to be written with information about all entries that failed
114 *       validation.</LI>
115 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
116 *       concurrent threads to use when processing the LDIF.  If this is not
117 *       provided, then a default of one thread will be used.</LI>
118 *   <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation
119 *       process should ignore validation failures due to entries that contain
120 *       object classes not defined in the server schema.</LI>
121 *   <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process
122 *       should ignore validation failures due to entries that contain
123 *       attributes not defined in the server schema.</LI>
124 *   <LI>"--ignoreMalformedDNs" -- indicates that the validation process should
125 *       ignore validation failures due to entries with malformed DNs.</LI>
126 *   <LI>"--ignoreMissingRDNValues" -- indicates that the validation process
127 *       should ignore validation failures due to entries that contain an RDN
128 *       attribute value that is not present in the set of entry
129 *       attributes.</LI>
130 *   <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation
131 *       process should ignore validation failures due to entries that either do
132 *       not have a structural object class or that have multiple structural
133 *       object classes.</LI>
134 *   <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation
135 *       process should ignore validation failures due to entries containing
136 *       auxiliary classes that are not allowed by a DIT content rule, or
137 *       abstract classes that are not subclassed by an auxiliary or structural
138 *       class contained in the entry.</LI>
139 *   <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process
140 *       should ignore validation failures due to entries including attributes
141 *       that are not allowed or are explicitly prohibited by a DIT content
142 *       rule.</LI>
143 *   <LI>"--ignoreMissingAttributes" -- indicates that the validation process
144 *       should ignore validation failures due to entries missing required
145 *       attributes.</LI>
146 *   <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation
147 *       process should ignore validation failures due to single-valued
148 *       attributes containing multiple values.</LI>
149 *   <LI>"--ignoreAttributeSyntax" -- indicates that the validation process
150 *       should ignore validation failures due to attribute values which violate
151 *       the associated attribute syntax.</LI>
152 *   <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation
153 *       process should ignore validation failures due to attribute values which
154 *       violate the associated attribute syntax, but only for the specified
155 *       attribute types.</LI>
156 *   <LI>"--ignoreNameForms" -- indicates that the validation process should
157 *       ignore validation failures due to name form violations (in which the
158 *       entry's RDN does not comply with the associated name form).</LI>
159 * </UL>
160 */
161@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
162public final class ValidateLDIF
163       extends LDAPCommandLineTool
164       implements LDIFReaderEntryTranslator
165{
166  /**
167   * The end-of-line character for this platform.
168   */
169  private static final String EOL =
170       StaticUtils.getSystemProperty("line.separator", "\n");
171
172
173
174  // The arguments used by this program.
175  private BooleanArgument ignoreDuplicateValues;
176  private BooleanArgument ignoreUndefinedObjectClasses;
177  private BooleanArgument ignoreUndefinedAttributes;
178  private BooleanArgument ignoreMalformedDNs;
179  private BooleanArgument ignoreMissingRDNValues;
180  private BooleanArgument ignoreMissingSuperiorObjectClasses;
181  private BooleanArgument ignoreStructuralObjectClasses;
182  private BooleanArgument ignoreProhibitedObjectClasses;
183  private BooleanArgument ignoreProhibitedAttributes;
184  private BooleanArgument ignoreMissingAttributes;
185  private BooleanArgument ignoreSingleValuedAttributes;
186  private BooleanArgument ignoreAttributeSyntax;
187  private BooleanArgument ignoreNameForms;
188  private BooleanArgument isCompressed;
189  private FileArgument    schemaDirectory;
190  private FileArgument    ldifFile;
191  private FileArgument    rejectFile;
192  private FileArgument    encryptionPassphraseFile;
193  private IntegerArgument numThreads;
194  private StringArgument  ignoreSyntaxViolationsForAttribute;
195
196  // The counter used to keep track of the number of entries processed.
197  private final AtomicLong entriesProcessed = new AtomicLong(0L);
198
199  // The counter used to keep track of the number of entries that could not be
200  // parsed as valid entries.
201  private final AtomicLong malformedEntries = new AtomicLong(0L);
202
203  // The entry validator that will be used to validate the entries.
204  private EntryValidator entryValidator;
205
206  // The LDIF writer that will be used to write rejected entries.
207  private LDIFWriter rejectWriter;
208
209
210
211  /**
212   * Parse the provided command line arguments and make the appropriate set of
213   * changes.
214   *
215   * @param  args  The command line arguments provided to this program.
216   */
217  public static void main(final String[] args)
218  {
219    final ResultCode resultCode = main(args, System.out, System.err);
220    if (resultCode != ResultCode.SUCCESS)
221    {
222      System.exit(resultCode.intValue());
223    }
224  }
225
226
227
228  /**
229   * Parse the provided command line arguments and make the appropriate set of
230   * changes.
231   *
232   * @param  args       The command line arguments provided to this program.
233   * @param  outStream  The output stream to which standard out should be
234   *                    written.  It may be {@code null} if output should be
235   *                    suppressed.
236   * @param  errStream  The output stream to which standard error should be
237   *                    written.  It may be {@code null} if error messages
238   *                    should be suppressed.
239   *
240   * @return  A result code indicating whether the processing was successful.
241   */
242  public static ResultCode main(final String[] args,
243                                final OutputStream outStream,
244                                final OutputStream errStream)
245  {
246    final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream);
247    return validateLDIF.runTool(args);
248  }
249
250
251
252  /**
253   * Creates a new instance of this tool.
254   *
255   * @param  outStream  The output stream to which standard out should be
256   *                    written.  It may be {@code null} if output should be
257   *                    suppressed.
258   * @param  errStream  The output stream to which standard error should be
259   *                    written.  It may be {@code null} if error messages
260   *                    should be suppressed.
261   */
262  public ValidateLDIF(final OutputStream outStream,
263                      final OutputStream errStream)
264  {
265    super(outStream, errStream);
266  }
267
268
269
270  /**
271   * Retrieves the name for this tool.
272   *
273   * @return  The name for this tool.
274   */
275  @Override()
276  public String getToolName()
277  {
278    return "validate-ldif";
279  }
280
281
282
283  /**
284   * Retrieves the description for this tool.
285   *
286   * @return  The description for this tool.
287   */
288  @Override()
289  public String getToolDescription()
290  {
291    return "Validate the contents of an LDIF file " +
292           "against the server schema.";
293  }
294
295
296
297  /**
298   * Retrieves the version string for this tool.
299   *
300   * @return  The version string for this tool.
301   */
302  @Override()
303  public String getToolVersion()
304  {
305    return Version.NUMERIC_VERSION_STRING;
306  }
307
308
309
310  /**
311   * Indicates whether this tool should provide support for an interactive mode,
312   * in which the tool offers a mode in which the arguments can be provided in
313   * a text-driven menu rather than requiring them to be given on the command
314   * line.  If interactive mode is supported, it may be invoked using the
315   * "--interactive" argument.  Alternately, if interactive mode is supported
316   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
317   * interactive mode may be invoked by simply launching the tool without any
318   * arguments.
319   *
320   * @return  {@code true} if this tool supports interactive mode, or
321   *          {@code false} if not.
322   */
323  @Override()
324  public boolean supportsInteractiveMode()
325  {
326    return true;
327  }
328
329
330
331  /**
332   * Indicates whether this tool defaults to launching in interactive mode if
333   * the tool is invoked without any command-line arguments.  This will only be
334   * used if {@link #supportsInteractiveMode()} returns {@code true}.
335   *
336   * @return  {@code true} if this tool defaults to using interactive mode if
337   *          launched without any command-line arguments, or {@code false} if
338   *          not.
339   */
340  @Override()
341  public boolean defaultsToInteractiveMode()
342  {
343    return true;
344  }
345
346
347
348  /**
349   * Indicates whether this tool should provide arguments for redirecting output
350   * to a file.  If this method returns {@code true}, then the tool will offer
351   * an "--outputFile" argument that will specify the path to a file to which
352   * all standard output and standard error content will be written, and it will
353   * also offer a "--teeToStandardOut" argument that can only be used if the
354   * "--outputFile" argument is present and will cause all output to be written
355   * to both the specified output file and to standard output.
356   *
357   * @return  {@code true} if this tool should provide arguments for redirecting
358   *          output to a file, or {@code false} if not.
359   */
360  @Override()
361  protected boolean supportsOutputFile()
362  {
363    return true;
364  }
365
366
367
368  /**
369   * Indicates whether this tool should default to interactively prompting for
370   * the bind password if a password is required but no argument was provided
371   * to indicate how to get the password.
372   *
373   * @return  {@code true} if this tool should default to interactively
374   *          prompting for the bind password, or {@code false} if not.
375   */
376  @Override()
377  protected boolean defaultToPromptForBindPassword()
378  {
379    return true;
380  }
381
382
383
384  /**
385   * Indicates whether this tool supports the use of a properties file for
386   * specifying default values for arguments that aren't specified on the
387   * command line.
388   *
389   * @return  {@code true} if this tool supports the use of a properties file
390   *          for specifying default values for arguments that aren't specified
391   *          on the command line, or {@code false} if not.
392   */
393  @Override()
394  public boolean supportsPropertiesFile()
395  {
396    return true;
397  }
398
399
400
401  /**
402   * Indicates whether the LDAP-specific arguments should include alternate
403   * versions of all long identifiers that consist of multiple words so that
404   * they are available in both camelCase and dash-separated versions.
405   *
406   * @return  {@code true} if this tool should provide multiple versions of
407   *          long identifiers for LDAP-specific arguments, or {@code false} if
408   *          not.
409   */
410  @Override()
411  protected boolean includeAlternateLongIdentifiers()
412  {
413    return true;
414  }
415
416
417
418  /**
419   * Indicates whether this tool should provide a command-line argument that
420   * allows for low-level SSL debugging.  If this returns {@code true}, then an
421   * "--enableSSLDebugging}" argument will be added that sets the
422   * "javax.net.debug" system property to "all" before attempting any
423   * communication.
424   *
425   * @return  {@code true} if this tool should offer an "--enableSSLDebugging"
426   *          argument, or {@code false} if not.
427   */
428  @Override()
429  protected boolean supportsSSLDebugging()
430  {
431    return true;
432  }
433
434
435
436  /**
437   * Adds the arguments used by this program that aren't already provided by the
438   * generic {@code LDAPCommandLineTool} framework.
439   *
440   * @param  parser  The argument parser to which the arguments should be added.
441   *
442   * @throws  ArgumentException  If a problem occurs while adding the arguments.
443   */
444  @Override()
445  public void addNonLDAPArguments(final ArgumentParser parser)
446         throws ArgumentException
447  {
448    String description = "The path to the LDIF file to process.  The tool " +
449         "will automatically attempt to detect whether the file is " +
450         "encrypted or compressed.";
451    ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description,
452                                true, true, true, false);
453    ldifFile.addLongIdentifier("ldif-file", true);
454    parser.addArgument(ldifFile);
455
456
457    // Add an argument that makes it possible to read a compressed LDIF file.
458    // Note that this argument is no longer needed for dealing with compressed
459    // files, since the tool will automatically detect whether a file is
460    // compressed.  However, the argument is still provided for the purpose of
461    // backward compatibility.
462    description = "Indicates that the specified LDIF file is compressed " +
463                  "using gzip compression.";
464    isCompressed = new BooleanArgument('c', "isCompressed", description);
465    isCompressed.addLongIdentifier("is-compressed", true);
466    isCompressed.setHidden(true);
467    parser.addArgument(isCompressed);
468
469
470    // Add an argument that indicates that the tool should read the encryption
471    // passphrase from a file.
472    description = "Indicates that the specified LDIF file is encrypted and " +
473         "that the encryption passphrase is contained in the specified " +
474         "file.  If the LDIF data is encrypted and this argument is not " +
475         "provided, then the tool will interactively prompt for the " +
476         "encryption passphrase.";
477    encryptionPassphraseFile = new FileArgument(null,
478         "encryptionPassphraseFile", false, 1, null, description, true, true,
479         true, false);
480    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
481         true);
482    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
483    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
484         true);
485    parser.addArgument(encryptionPassphraseFile);
486
487
488    description = "The path to the file to which rejected entries should be " +
489                  "written.";
490    rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}",
491                                  description, false, true, true, false);
492    rejectFile.addLongIdentifier("reject-file", true);
493    parser.addArgument(rejectFile);
494
495    description = "The path to a directory containing one or more LDIF files " +
496                  "with the schema information to use.  If this is provided, " +
497                  "then no LDAP communication will be performed.";
498    schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1,
499         "{path}", description, true, true, false, true);
500    schemaDirectory.addLongIdentifier("schema-directory", true);
501    parser.addArgument(schemaDirectory);
502
503    description = "The number of threads to use when processing the LDIF file.";
504    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
505         description, 1, Integer.MAX_VALUE, 1);
506    numThreads.addLongIdentifier("num-threads", true);
507    parser.addArgument(numThreads);
508
509    description = "Ignore validation failures due to entries containing " +
510                  "duplicate values for the same attribute.";
511    ignoreDuplicateValues =
512         new BooleanArgument(null, "ignoreDuplicateValues", description);
513    ignoreDuplicateValues.setArgumentGroupName(
514         "Validation Strictness Arguments");
515    ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values", true);
516    parser.addArgument(ignoreDuplicateValues);
517
518    description = "Ignore validation failures due to object classes not " +
519                  "defined in the schema.";
520    ignoreUndefinedObjectClasses =
521         new BooleanArgument(null, "ignoreUndefinedObjectClasses", description);
522    ignoreUndefinedObjectClasses.setArgumentGroupName(
523         "Validation Strictness Arguments");
524    ignoreUndefinedObjectClasses.addLongIdentifier(
525         "ignore-undefined-object-classes", true);
526    parser.addArgument(ignoreUndefinedObjectClasses);
527
528    description = "Ignore validation failures due to attributes not defined " +
529                  "in the schema.";
530    ignoreUndefinedAttributes =
531         new BooleanArgument(null, "ignoreUndefinedAttributes", description);
532    ignoreUndefinedAttributes.setArgumentGroupName(
533         "Validation Strictness Arguments");
534    ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes",
535         true);
536    parser.addArgument(ignoreUndefinedAttributes);
537
538    description = "Ignore validation failures due to entries with malformed " +
539                  "DNs.";
540    ignoreMalformedDNs =
541         new BooleanArgument(null, "ignoreMalformedDNs", description);
542    ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments");
543    ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns", true);
544    parser.addArgument(ignoreMalformedDNs);
545
546    description = "Ignore validation failures due to entries with RDN " +
547                  "attribute values that are missing from the set of entry " +
548                  "attributes.";
549    ignoreMissingRDNValues =
550         new BooleanArgument(null, "ignoreMissingRDNValues", description);
551    ignoreMissingRDNValues.setArgumentGroupName(
552         "Validation Strictness Arguments");
553    ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values", true);
554    parser.addArgument(ignoreMissingRDNValues);
555
556    description = "Ignore validation failures due to entries without exactly " +
557                  "structural object class.";
558    ignoreStructuralObjectClasses =
559         new BooleanArgument(null, "ignoreStructuralObjectClasses",
560                             description);
561    ignoreStructuralObjectClasses.setArgumentGroupName(
562         "Validation Strictness Arguments");
563    ignoreStructuralObjectClasses.addLongIdentifier(
564         "ignore-structural-object-classes", true);
565    parser.addArgument(ignoreStructuralObjectClasses);
566
567    description = "Ignore validation failures due to entries with object " +
568                  "classes that are not allowed.";
569    ignoreProhibitedObjectClasses =
570         new BooleanArgument(null, "ignoreProhibitedObjectClasses",
571                             description);
572    ignoreProhibitedObjectClasses.setArgumentGroupName(
573         "Validation Strictness Arguments");
574    ignoreProhibitedObjectClasses.addLongIdentifier(
575         "ignore-prohibited-object-classes", true);
576    parser.addArgument(ignoreProhibitedObjectClasses);
577
578    description = "Ignore validation failures due to entries that are " +
579                  "one or more superior object classes.";
580    ignoreMissingSuperiorObjectClasses =
581         new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses",
582              description);
583    ignoreMissingSuperiorObjectClasses.setArgumentGroupName(
584         "Validation Strictness Arguments");
585    ignoreMissingSuperiorObjectClasses.addLongIdentifier(
586         "ignore-missing-superior-object-classes", true);
587    parser.addArgument(ignoreMissingSuperiorObjectClasses);
588
589    description = "Ignore validation failures due to entries with attributes " +
590                  "that are not allowed.";
591    ignoreProhibitedAttributes =
592         new BooleanArgument(null, "ignoreProhibitedAttributes", description);
593    ignoreProhibitedAttributes.setArgumentGroupName(
594         "Validation Strictness Arguments");
595    ignoreProhibitedAttributes.addLongIdentifier(
596         "ignore-prohibited-attributes", true);
597    parser.addArgument(ignoreProhibitedAttributes);
598
599    description = "Ignore validation failures due to entries missing " +
600                  "required attributes.";
601    ignoreMissingAttributes =
602         new BooleanArgument(null, "ignoreMissingAttributes", description);
603    ignoreMissingAttributes.setArgumentGroupName(
604         "Validation Strictness Arguments");
605    ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes",
606         true);
607    parser.addArgument(ignoreMissingAttributes);
608
609    description = "Ignore validation failures due to entries with multiple " +
610                  "values for single-valued attributes.";
611    ignoreSingleValuedAttributes =
612         new BooleanArgument(null, "ignoreSingleValuedAttributes", description);
613    ignoreSingleValuedAttributes.setArgumentGroupName(
614         "Validation Strictness Arguments");
615    ignoreSingleValuedAttributes.addLongIdentifier(
616         "ignore-single-valued-attributes", true);
617    parser.addArgument(ignoreSingleValuedAttributes);
618
619    description = "Ignore validation failures due to entries with attribute " +
620                  "values that violate their associated syntax.  If this is " +
621                  "provided, then no attribute syntax violations will be " +
622                  "flagged.  If this is not provided, then all attribute " +
623                  "syntax violations will be flagged except for violations " +
624                  "in those attributes excluded by the " +
625                  "--ignoreSyntaxViolationsForAttribute argument.";
626    ignoreAttributeSyntax =
627         new BooleanArgument(null, "ignoreAttributeSyntax", description);
628    ignoreAttributeSyntax.setArgumentGroupName(
629         "Validation Strictness Arguments");
630    ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax", true);
631    parser.addArgument(ignoreAttributeSyntax);
632
633    description = "The name or OID of an attribute for which to ignore " +
634                  "validation failures due to violations of the associated " +
635                  "attribute syntax.  This argument can only be used if the " +
636                  "--ignoreAttributeSyntax argument is not provided.";
637    ignoreSyntaxViolationsForAttribute = new StringArgument(null,
638         "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description);
639    ignoreSyntaxViolationsForAttribute.setArgumentGroupName(
640         "Validation Strictness Arguments");
641    ignoreSyntaxViolationsForAttribute.addLongIdentifier(
642         "ignore-syntax-violations-for-attribute", true);
643    parser.addArgument(ignoreSyntaxViolationsForAttribute);
644
645    description = "Ignore validation failures due to entries with RDNs " +
646                  "that violate the associated name form definition.";
647    ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description);
648    ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments");
649    ignoreNameForms.addLongIdentifier("ignore-name-forms", true);
650    parser.addArgument(ignoreNameForms);
651
652
653    // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments
654    // cannot be used together.
655    parser.addExclusiveArgumentSet(ignoreAttributeSyntax,
656         ignoreSyntaxViolationsForAttribute);
657  }
658
659
660
661  /**
662   * Performs the actual processing for this tool.  In this case, it gets a
663   * connection to the directory server and uses it to retrieve the server
664   * schema.  It then reads the LDIF file and validates each entry accordingly.
665   *
666   * @return  The result code for the processing that was performed.
667   */
668  @Override()
669  public ResultCode doToolProcessing()
670  {
671    // Get the connection to the directory server and use it to read the schema.
672    final Schema schema;
673    if (schemaDirectory.isPresent())
674    {
675      final File schemaDir = schemaDirectory.getValue();
676
677      try
678      {
679        final TreeMap<String,File> fileMap = new TreeMap<>();
680        for (final File f : schemaDir.listFiles())
681        {
682          final String name = f.getName();
683          if (f.isFile() && name.endsWith(".ldif"))
684          {
685            fileMap.put(name, f);
686          }
687        }
688
689        if (fileMap.isEmpty())
690        {
691          err("No LDIF files found in directory " +
692              schemaDir.getAbsolutePath());
693          return ResultCode.PARAM_ERROR;
694        }
695
696        final ArrayList<File> fileList = new ArrayList<>(fileMap.values());
697        schema = Schema.getSchema(fileList);
698      }
699      catch (final Exception e)
700      {
701        Debug.debugException(e);
702        err("Unable to read schema from files in directory " +
703            schemaDir.getAbsolutePath() + ":  " +
704             StaticUtils.getExceptionMessage(e));
705        return ResultCode.LOCAL_ERROR;
706      }
707    }
708    else
709    {
710      try
711      {
712        final LDAPConnection connection = getConnection();
713        schema = connection.getSchema();
714        connection.close();
715      }
716      catch (final LDAPException le)
717      {
718        Debug.debugException(le);
719        err("Unable to connect to the directory server and read the schema:  ",
720            le.getMessage());
721        return le.getResultCode();
722      }
723    }
724
725
726    // Get the encryption passphrase, if it was provided.
727    String encryptionPassphrase = null;
728    if (encryptionPassphraseFile.isPresent())
729    {
730      try
731      {
732        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
733             encryptionPassphraseFile.getValue());
734      }
735      catch (final LDAPException e)
736      {
737        Debug.debugException(e);
738        err(e.getMessage());
739        return e.getResultCode();
740      }
741    }
742
743
744    // Create the entry validator and initialize its configuration.
745    entryValidator = new EntryValidator(schema);
746    entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent());
747    entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent());
748    entryValidator.setCheckEntryMissingRDNValues(
749         !ignoreMissingRDNValues.isPresent());
750    entryValidator.setCheckMissingAttributes(
751         !ignoreMissingAttributes.isPresent());
752    entryValidator.setCheckNameForms(!ignoreNameForms.isPresent());
753    entryValidator.setCheckProhibitedAttributes(
754         !ignoreProhibitedAttributes.isPresent());
755    entryValidator.setCheckProhibitedObjectClasses(
756         !ignoreProhibitedObjectClasses.isPresent());
757    entryValidator.setCheckMissingSuperiorObjectClasses(
758         !ignoreMissingSuperiorObjectClasses.isPresent());
759    entryValidator.setCheckSingleValuedAttributes(
760         !ignoreSingleValuedAttributes.isPresent());
761    entryValidator.setCheckStructuralObjectClasses(
762         !ignoreStructuralObjectClasses.isPresent());
763    entryValidator.setCheckUndefinedAttributes(
764         !ignoreUndefinedAttributes.isPresent());
765    entryValidator.setCheckUndefinedObjectClasses(
766         !ignoreUndefinedObjectClasses.isPresent());
767
768    if (ignoreSyntaxViolationsForAttribute.isPresent())
769    {
770      entryValidator.setIgnoreSyntaxViolationAttributeTypes(
771           ignoreSyntaxViolationsForAttribute.getValues());
772    }
773
774
775    // Create an LDIF reader that can be used to read through the LDIF file.
776    final LDIFReader ldifReader;
777    rejectWriter = null;
778    try
779    {
780      InputStream inputStream = new FileInputStream(ldifFile.getValue());
781
782      inputStream = ToolUtils.getPossiblyPassphraseEncryptedInputStream(
783           inputStream, encryptionPassphrase, false,
784           "LDIF file '" + ldifFile.getValue().getPath() +
785                "' is encrypted.  Please enter the encryption passphrase:",
786             "ERROR:  The provided passphrase was incorrect.",
787             getOut(), getErr()).getFirst();
788
789      if (isCompressed.isPresent())
790      {
791        inputStream = new GZIPInputStream(inputStream);
792      }
793      else
794      {
795        inputStream =
796             ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream);
797      }
798
799      ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this);
800    }
801    catch (final Exception e)
802    {
803      Debug.debugException(e);
804      err("Unable to open the LDIF reader:  ",
805           StaticUtils.getExceptionMessage(e));
806      return ResultCode.LOCAL_ERROR;
807    }
808
809    ldifReader.setSchema(schema);
810    if (ignoreDuplicateValues.isPresent())
811    {
812      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP);
813    }
814    else
815    {
816      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT);
817    }
818
819    try
820    {
821      // Create an LDIF writer that can be used to write information about
822      // rejected entries.
823      try
824      {
825        if (rejectFile.isPresent())
826        {
827          rejectWriter = new LDIFWriter(rejectFile.getValue());
828        }
829      }
830      catch (final Exception e)
831      {
832        Debug.debugException(e);
833        err("Unable to create the reject writer:  ",
834             StaticUtils.getExceptionMessage(e));
835        return ResultCode.LOCAL_ERROR;
836      }
837
838      ResultCode resultCode = ResultCode.SUCCESS;
839      while (true)
840      {
841        try
842        {
843          final Entry e = ldifReader.readEntry();
844          if (e == null)
845          {
846            // Because we're performing parallel processing and returning null
847            // from the translate method, LDIFReader.readEntry() should never
848            // return a non-null value.  However, it can throw an LDIFException
849            // if it encounters an invalid entry, or an IOException if there's
850            // a problem reading from the file, so we should still iterate
851            // through all of the entries to catch and report on those problems.
852            break;
853          }
854        }
855        catch (final LDIFException le)
856        {
857          Debug.debugException(le);
858          malformedEntries.incrementAndGet();
859
860          if (resultCode == ResultCode.SUCCESS)
861          {
862            resultCode = ResultCode.DECODING_ERROR;
863          }
864
865          if (rejectWriter != null)
866          {
867            try
868            {
869              rejectWriter.writeComment(
870                   "Unable to parse an entry read from LDIF:", false, false);
871              if (le.mayContinueReading())
872              {
873                rejectWriter.writeComment(
874                     StaticUtils.getExceptionMessage(le), false, true);
875              }
876              else
877              {
878                rejectWriter.writeComment(
879                     StaticUtils.getExceptionMessage(le), false,
880                     false);
881                rejectWriter.writeComment("Unable to continue LDIF processing.",
882                     false, true);
883                err("Aborting LDIF processing:  ",
884                     StaticUtils.getExceptionMessage(le));
885                return ResultCode.LOCAL_ERROR;
886              }
887            }
888            catch (final IOException ioe)
889            {
890              Debug.debugException(ioe);
891              err("Unable to write to the reject file:",
892                  StaticUtils.getExceptionMessage(ioe));
893              err("LDIF parse failure that triggered the rejection:  ",
894                  StaticUtils.getExceptionMessage(le));
895              return ResultCode.LOCAL_ERROR;
896            }
897          }
898        }
899        catch (final IOException ioe)
900        {
901          Debug.debugException(ioe);
902
903          if (rejectWriter != null)
904          {
905            try
906            {
907              rejectWriter.writeComment("I/O error reading from LDIF:", false,
908                   false);
909              rejectWriter.writeComment(StaticUtils.getExceptionMessage(ioe),
910                   false, true);
911              return ResultCode.LOCAL_ERROR;
912            }
913            catch (final Exception ex)
914            {
915              Debug.debugException(ex);
916              err("I/O error reading from LDIF:",
917                   StaticUtils.getExceptionMessage(ioe));
918              return ResultCode.LOCAL_ERROR;
919            }
920          }
921        }
922      }
923
924      if (malformedEntries.get() > 0)
925      {
926        out(malformedEntries.get() + " entries were malformed and could not " +
927            "be read from the LDIF file.");
928      }
929
930      if (entryValidator.getInvalidEntries() > 0)
931      {
932        if (resultCode == ResultCode.SUCCESS)
933        {
934          resultCode = ResultCode.OBJECT_CLASS_VIOLATION;
935        }
936
937        for (final String s : entryValidator.getInvalidEntrySummary(true))
938        {
939          out(s);
940        }
941      }
942      else
943      {
944        if (malformedEntries.get() == 0)
945        {
946          out("No errors were encountered.");
947        }
948      }
949
950      return resultCode;
951    }
952    finally
953    {
954      try
955      {
956        ldifReader.close();
957      }
958      catch (final Exception e)
959      {
960        Debug.debugException(e);
961      }
962
963      try
964      {
965        if (rejectWriter != null)
966        {
967          rejectWriter.close();
968        }
969      }
970      catch (final Exception e)
971      {
972        Debug.debugException(e);
973      }
974    }
975  }
976
977
978
979  /**
980   * Examines the provided entry to determine whether it conforms to the
981   * server schema.
982   *
983   * @param  entry           The entry to be examined.
984   * @param  firstLineNumber The line number of the LDIF source on which the
985   *                         provided entry begins.
986   *
987   * @return  The updated entry.  This method will always return {@code null}
988   *          because all of the real processing needed for the entry is
989   *          performed in this method and the entry isn't needed any more
990   *          after this method is done.
991   */
992  @Override()
993  public Entry translate(final Entry entry, final long firstLineNumber)
994  {
995    final ArrayList<String> invalidReasons = new ArrayList<>(5);
996    if (! entryValidator.entryIsValid(entry, invalidReasons))
997    {
998      if (rejectWriter != null)
999      {
1000        synchronized (this)
1001        {
1002          try
1003          {
1004            rejectWriter.writeEntry(entry, listToString(invalidReasons));
1005          }
1006          catch (final IOException ioe)
1007          {
1008            Debug.debugException(ioe);
1009          }
1010        }
1011      }
1012    }
1013
1014    final long numEntries = entriesProcessed.incrementAndGet();
1015    if ((numEntries % 1000L) == 0L)
1016    {
1017      out("Processed ", numEntries, " entries.");
1018    }
1019
1020    return null;
1021  }
1022
1023
1024
1025  /**
1026   * Converts the provided list of strings into a single string.  It will
1027   * contain line breaks after all but the last element.
1028   *
1029   * @param  l  The list of strings to convert to a single string.
1030   *
1031   * @return  The string from the provided list, or {@code null} if the provided
1032   *          list is empty or {@code null}.
1033   */
1034  private static String listToString(final List<String> l)
1035  {
1036    if ((l == null) || (l.isEmpty()))
1037    {
1038      return null;
1039    }
1040
1041    final StringBuilder buffer = new StringBuilder();
1042    final Iterator<String> iterator = l.iterator();
1043    while (iterator.hasNext())
1044    {
1045      buffer.append(iterator.next());
1046      if (iterator.hasNext())
1047      {
1048        buffer.append(EOL);
1049      }
1050    }
1051
1052    return buffer.toString();
1053  }
1054
1055
1056
1057  /**
1058   * {@inheritDoc}
1059   */
1060  @Override()
1061  public LinkedHashMap<String[],String> getExampleUsages()
1062  {
1063    final LinkedHashMap<String[],String> examples =
1064         new LinkedHashMap<>(StaticUtils.computeMapCapacity(2));
1065
1066    String[] args =
1067    {
1068      "--hostname", "server.example.com",
1069      "--port", "389",
1070      "--ldifFile", "data.ldif",
1071      "--rejectFile", "rejects.ldif",
1072      "--numThreads", "4"
1073    };
1074    String description =
1075         "Validate the contents of the 'data.ldif' file using the schema " +
1076         "defined in the specified directory server using four concurrent " +
1077         "threads.  All types of validation will be performed, and " +
1078         "information about any errors will be written to the 'rejects.ldif' " +
1079         "file.";
1080    examples.put(args, description);
1081
1082
1083    args = new String[]
1084    {
1085      "--schemaDirectory", "/ds/config/schema",
1086      "--ldifFile", "data.ldif",
1087      "--rejectFile", "rejects.ldif",
1088      "--ignoreStructuralObjectClasses",
1089      "--ignoreAttributeSyntax"
1090    };
1091    description =
1092         "Validate the contents of the 'data.ldif' file using the schema " +
1093         "defined in LDIF files contained in the /ds/config/schema directory " +
1094         "using a single thread.  Any errors resulting from entries that do " +
1095         "not have exactly one structural object class or from values which " +
1096         "violate the syntax for their associated attribute types will be " +
1097         "ignored.  Information about any other failures will be written to " +
1098         "the 'rejects.ldif' file.";
1099    examples.put(args, description);
1100
1101    return examples;
1102  }
1103
1104
1105
1106  /**
1107   * @return EntryValidator
1108   *
1109   * Returns the EntryValidator
1110   */
1111  public EntryValidator getEntryValidator()
1112  {
1113    return entryValidator;
1114  }
1115}