001/*
002 * Copyright 2007-2020 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2007-2020 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2008-2020 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldif;
037
038
039
040import java.io.BufferedReader;
041import java.io.Closeable;
042import java.io.File;
043import java.io.FileInputStream;
044import java.io.InputStream;
045import java.io.InputStreamReader;
046import java.io.IOException;
047import java.nio.charset.StandardCharsets;
048import java.text.ParseException;
049import java.util.ArrayList;
050import java.util.Collection;
051import java.util.Iterator;
052import java.util.HashSet;
053import java.util.LinkedHashMap;
054import java.util.List;
055import java.util.Set;
056import java.util.concurrent.BlockingQueue;
057import java.util.concurrent.ArrayBlockingQueue;
058import java.util.concurrent.TimeUnit;
059import java.util.concurrent.atomic.AtomicBoolean;
060import java.nio.charset.Charset;
061
062import com.unboundid.asn1.ASN1OctetString;
063import com.unboundid.ldap.matchingrules.CaseIgnoreStringMatchingRule;
064import com.unboundid.ldap.matchingrules.MatchingRule;
065import com.unboundid.ldap.sdk.Attribute;
066import com.unboundid.ldap.sdk.Control;
067import com.unboundid.ldap.sdk.Entry;
068import com.unboundid.ldap.sdk.Modification;
069import com.unboundid.ldap.sdk.ModificationType;
070import com.unboundid.ldap.sdk.LDAPException;
071import com.unboundid.ldap.sdk.schema.AttributeTypeDefinition;
072import com.unboundid.ldap.sdk.schema.Schema;
073import com.unboundid.util.AggregateInputStream;
074import com.unboundid.util.Base64;
075import com.unboundid.util.Debug;
076import com.unboundid.util.LDAPSDKThreadFactory;
077import com.unboundid.util.StaticUtils;
078import com.unboundid.util.ThreadSafety;
079import com.unboundid.util.ThreadSafetyLevel;
080import com.unboundid.util.Validator;
081import com.unboundid.util.parallel.AsynchronousParallelProcessor;
082import com.unboundid.util.parallel.Result;
083import com.unboundid.util.parallel.ParallelProcessor;
084import com.unboundid.util.parallel.Processor;
085
086import static com.unboundid.ldif.LDIFMessages.*;
087
088/**
089 * This class provides an LDIF reader, which can be used to read and decode
090 * entries and change records from a data source using the LDAP Data Interchange
091 * Format as per <A HREF="http://www.ietf.org/rfc/rfc2849.txt">RFC 2849</A>.
092 * <BR>
093 * This class is not synchronized.  If multiple threads read from the
094 * LDIFReader, they must be synchronized externally.
095 * <BR><BR>
096 * <H2>Example</H2>
097 * The following example iterates through all entries contained in an LDIF file
098 * and attempts to add them to a directory server:
099 * <PRE>
100 * LDIFReader ldifReader = new LDIFReader(pathToLDIFFile);
101 *
102 * int entriesRead = 0;
103 * int entriesAdded = 0;
104 * int errorsEncountered = 0;
105 * while (true)
106 * {
107 *   Entry entry;
108 *   try
109 *   {
110 *     entry = ldifReader.readEntry();
111 *     if (entry == null)
112 *     {
113 *       // All entries have been read.
114 *       break;
115 *     }
116 *
117 *     entriesRead++;
118 *   }
119 *   catch (LDIFException le)
120 *   {
121 *     errorsEncountered++;
122 *     if (le.mayContinueReading())
123 *     {
124 *       // A recoverable error occurred while attempting to read a change
125 *       // record, at or near line number le.getLineNumber()
126 *       // The entry will be skipped, but we'll try to keep reading from the
127 *       // LDIF file.
128 *       continue;
129 *     }
130 *     else
131 *     {
132 *       // An unrecoverable error occurred while attempting to read an entry
133 *       // at or near line number le.getLineNumber()
134 *       // No further LDIF processing will be performed.
135 *       break;
136 *     }
137 *   }
138 *   catch (IOException ioe)
139 *   {
140 *     // An I/O error occurred while attempting to read from the LDIF file.
141 *     // No further LDIF processing will be performed.
142 *     errorsEncountered++;
143 *     break;
144 *   }
145 *
146 *   LDAPResult addResult;
147 *   try
148 *   {
149 *     addResult = connection.add(entry);
150 *     // If we got here, then the change should have been processed
151 *     // successfully.
152 *     entriesAdded++;
153 *   }
154 *   catch (LDAPException le)
155 *   {
156 *     // If we got here, then the change attempt failed.
157 *     addResult = le.toLDAPResult();
158 *     errorsEncountered++;
159 *   }
160 * }
161 *
162 * ldifReader.close();
163 * </PRE>
164 */
165@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
166public final class LDIFReader
167       implements Closeable
168{
169  /**
170   * The default buffer size (128KB) that will be used when reading from the
171   * data source.
172   */
173  public static final int DEFAULT_BUFFER_SIZE = 128 * 1024;
174
175
176
177  /*
178   * When processing asynchronously, this determines how many of the allocated
179   * worker threads are used to parse each batch of read entries.
180   */
181  private static final int ASYNC_MIN_PER_PARSING_THREAD = 3;
182
183
184
185  /**
186   * When processing asynchronously, this specifies the size of the pending and
187   * completed queues.
188   */
189  private static final int ASYNC_QUEUE_SIZE = 500;
190
191
192
193  /**
194   * Special entry used internally to signal that the LDIFReaderEntryTranslator
195   * has signalled that a read Entry should be skipped by returning null,
196   * which normally implies EOF.
197   */
198  private static final Entry SKIP_ENTRY = new Entry("cn=skipped");
199
200
201
202  /**
203   * The default base path that will be prepended to relative paths.  It will
204   * end with a trailing slash.
205   */
206  private static final String DEFAULT_RELATIVE_BASE_PATH;
207  static
208  {
209    final File currentDir;
210    final String currentDirString = StaticUtils.getSystemProperty("user.dir");
211    if (currentDirString == null)
212    {
213      currentDir = new File(".");
214    }
215    else
216    {
217      currentDir = new File(currentDirString);
218    }
219
220    final String currentDirAbsolutePath = currentDir.getAbsolutePath();
221    if (currentDirAbsolutePath.endsWith(File.separator))
222    {
223      DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath;
224    }
225    else
226    {
227      DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath + File.separator;
228    }
229  }
230
231
232
233  // The buffered reader that will be used to read LDIF data.
234  private final BufferedReader reader;
235
236  // The behavior that should be exhibited when encountering duplicate attribute
237  // values.
238  private volatile DuplicateValueBehavior duplicateValueBehavior;
239
240  // A line number counter.
241  private long lineNumberCounter = 0;
242
243  // The change record translator to use, if any.
244  private final LDIFReaderChangeRecordTranslator changeRecordTranslator;
245
246  // The entry translator to use, if any.
247  private final LDIFReaderEntryTranslator entryTranslator;
248
249  // The schema that will be used when processing, if applicable.
250  private Schema schema;
251
252  // Specifies the base path that will be prepended to relative paths for file
253  // URLs.
254  private volatile String relativeBasePath;
255
256  // The behavior that should be exhibited with regard to illegal trailing
257  // spaces in attribute values.
258  private volatile TrailingSpaceBehavior trailingSpaceBehavior;
259
260  // True iff we are processing asynchronously.
261  private final boolean isAsync;
262
263  //
264  // The following only apply to asynchronous processing.
265  //
266
267  // Parses entries asynchronously.
268  private final AsynchronousParallelProcessor<UnparsedLDIFRecord,LDIFRecord>
269       asyncParser;
270
271  // Set to true when the end of the input is reached.
272  private final AtomicBoolean asyncParsingComplete;
273
274  // The records that have been read and parsed.
275  private final BlockingQueue<Result<UnparsedLDIFRecord,LDIFRecord>>
276       asyncParsedRecords;
277
278
279
280  /**
281   * Creates a new LDIF reader that will read data from the specified file.
282   *
283   * @param  path  The path to the file from which the data is to be read.  It
284   *               must not be {@code null}.
285   *
286   * @throws  IOException  If a problem occurs while opening the file for
287   *                       reading.
288   */
289  public LDIFReader(final String path)
290         throws IOException
291  {
292    this(new FileInputStream(path));
293  }
294
295
296
297  /**
298   * Creates a new LDIF reader that will read data from the specified file
299   * and parses the LDIF records asynchronously using the specified number of
300   * threads.
301   *
302   * @param  path  The path to the file from which the data is to be read.  It
303   *               must not be {@code null}.
304   * @param  numParseThreads  If this value is greater than zero, then the
305   *                          specified number of threads will be used to
306   *                          asynchronously read and parse the LDIF file.
307   *
308   * @throws  IOException  If a problem occurs while opening the file for
309   *                       reading.
310   *
311   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
312   *      constructor for more details about asynchronous processing.
313   */
314  public LDIFReader(final String path, final int numParseThreads)
315         throws IOException
316  {
317    this(new FileInputStream(path), numParseThreads);
318  }
319
320
321
322  /**
323   * Creates a new LDIF reader that will read data from the specified file.
324   *
325   * @param  file  The file from which the data is to be read.  It must not be
326   *               {@code null}.
327   *
328   * @throws  IOException  If a problem occurs while opening the file for
329   *                       reading.
330   */
331  public LDIFReader(final File file)
332         throws IOException
333  {
334    this(new FileInputStream(file));
335  }
336
337
338
339  /**
340   * Creates a new LDIF reader that will read data from the specified file
341   * and optionally parses the LDIF records asynchronously using the specified
342   * number of threads.
343   *
344   * @param  file             The file from which the data is to be read.  It
345   *                          must not be {@code null}.
346   * @param  numParseThreads  If this value is greater than zero, then the
347   *                          specified number of threads will be used to
348   *                          asynchronously read and parse the LDIF file.
349   *
350   * @throws  IOException  If a problem occurs while opening the file for
351   *                       reading.
352   */
353  public LDIFReader(final File file, final int numParseThreads)
354         throws IOException
355  {
356    this(new FileInputStream(file), numParseThreads);
357  }
358
359
360
361  /**
362   * Creates a new LDIF reader that will read data from the specified files in
363   * the order in which they are provided and optionally parses the LDIF records
364   * asynchronously using the specified number of threads.
365   *
366   * @param  files            The files from which the data is to be read.  It
367   *                          must not be {@code null} or empty.
368   * @param  numParseThreads  If this value is greater than zero, then the
369   *                          specified number of threads will be used to
370   *                          asynchronously read and parse the LDIF file.
371   * @param entryTranslator   The LDIFReaderEntryTranslator to apply to entries
372   *                          before they are returned.  This is normally
373   *                          {@code null}, which causes entries to be returned
374   *                          unaltered. This is particularly useful when
375   *                          parsing the input file in parallel because the
376   *                          entry translation is also done in parallel.
377   *
378   * @throws  IOException  If a problem occurs while opening the file for
379   *                       reading.
380   */
381  public LDIFReader(final File[] files, final int numParseThreads,
382                    final LDIFReaderEntryTranslator entryTranslator)
383         throws IOException
384  {
385    this(files, numParseThreads, entryTranslator, null);
386  }
387
388
389
390  /**
391   * Creates a new LDIF reader that will read data from the specified files in
392   * the order in which they are provided and optionally parses the LDIF records
393   * asynchronously using the specified number of threads.
394   *
395   * @param  files                   The files from which the data is to be
396   *                                 read.  It must not be {@code null} or
397   *                                 empty.
398   * @param  numParseThreads         If this value is greater than zero, then
399   *                                 the specified number of threads will be
400   *                                 used to asynchronously read and parse the
401   *                                 LDIF file.
402   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
403   *                                 entries before they are returned.  This is
404   *                                 normally {@code null}, which causes entries
405   *                                 to be returned unaltered.  This is
406   *                                 particularly useful when parsing the input
407   *                                 file in parallel because the entry
408   *                                 translation is also done in parallel.
409   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
410   *                                 apply to change records before they are
411   *                                 returned.  This is normally {@code null},
412   *                                 which causes change records to be returned
413   *                                 unaltered.  This is particularly useful
414   *                                 when parsing the input file in parallel
415   *                                 because the change record translation is
416   *                                 also done in parallel.
417   *
418   * @throws  IOException  If a problem occurs while opening the file for
419   *                       reading.
420   */
421  public LDIFReader(final File[] files, final int numParseThreads,
422              final LDIFReaderEntryTranslator entryTranslator,
423              final LDIFReaderChangeRecordTranslator changeRecordTranslator)
424         throws IOException
425  {
426    this(files, numParseThreads, entryTranslator, changeRecordTranslator,
427         "UTF-8");
428  }
429
430
431
432  /**
433   * Creates a new LDIF reader that will read data from the specified files in
434   * the order in which they are provided and optionally parses the LDIF records
435   * asynchronously using the specified number of threads.
436   *
437   * @param  files                   The files from which the data is to be
438   *                                 read.  It must not be {@code null} or
439   *                                 empty.
440   * @param  numParseThreads         If this value is greater than zero, then
441   *                                 the specified number of threads will be
442   *                                 used to asynchronously read and parse the
443   *                                 LDIF file.
444   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
445   *                                 entries before they are returned.  This is
446   *                                 normally {@code null}, which causes entries
447   *                                 to be returned unaltered.  This is
448   *                                 particularly useful when parsing the input
449   *                                 file in parallel because the entry
450   *                                 translation is also done in parallel.
451   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
452   *                                 apply to change records before they are
453   *                                 returned.  This is normally {@code null},
454   *                                 which causes change records to be returned
455   *                                 unaltered.  This is particularly useful
456   *                                 when parsing the input file in parallel
457   *                                 because the change record translation is
458   *                                 also done in parallel.
459   * @param  characterSet            The character set to use when reading from
460   *                                 the input stream.  It must not be
461   *                                 {@code null}.
462   *
463   * @throws  IOException  If a problem occurs while opening the file for
464   *                       reading.
465   */
466  public LDIFReader(final File[] files, final int numParseThreads,
467              final LDIFReaderEntryTranslator entryTranslator,
468              final LDIFReaderChangeRecordTranslator changeRecordTranslator,
469              final String characterSet)
470         throws IOException
471  {
472    this(createAggregateInputStream(files), numParseThreads, entryTranslator,
473         changeRecordTranslator, characterSet);
474  }
475
476
477
478  /**
479   * Creates a new aggregate input stream that will read data from the specified
480   * files.  If there are multiple files, then a "padding" file will be inserted
481   * between them to ensure that there is at least one blank line between the
482   * end of one file and the beginning of another.
483   *
484   * @param  files  The files from which the data is to be read.  It must not be
485   *                {@code null} or empty.
486   *
487   * @return  The input stream to use to read data from the provided files.
488   *
489   * @throws  IOException  If a problem is encountered while attempting to
490   *                       create the input stream.
491   */
492  private static InputStream createAggregateInputStream(final File... files)
493          throws IOException
494  {
495    if (files.length == 0)
496    {
497      throw new IOException(ERR_READ_NO_LDIF_FILES.get());
498    }
499    else
500    {
501      return new AggregateInputStream(true, files);
502    }
503  }
504
505
506
507  /**
508   * Creates a new LDIF reader that will read data from the provided input
509   * stream.
510   *
511   * @param  inputStream  The input stream from which the data is to be read.
512   *                      It must not be {@code null}.
513   */
514  public LDIFReader(final InputStream inputStream)
515  {
516    this(inputStream, 0);
517  }
518
519
520
521  /**
522   * Creates a new LDIF reader that will read data from the specified stream
523   * and parses the LDIF records asynchronously using the specified number of
524   * threads.
525   *
526   * @param  inputStream  The input stream from which the data is to be read.
527   *                      It must not be {@code null}.
528   * @param  numParseThreads  If this value is greater than zero, then the
529   *                          specified number of threads will be used to
530   *                          asynchronously read and parse the LDIF file.
531   *
532   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
533   *      constructor for more details about asynchronous processing.
534   */
535  public LDIFReader(final InputStream inputStream, final int numParseThreads)
536  {
537    // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
538    this(new BufferedReader(
539         new InputStreamReader(inputStream, StandardCharsets.UTF_8),
540              DEFAULT_BUFFER_SIZE),
541         numParseThreads);
542  }
543
544
545
546  /**
547   * Creates a new LDIF reader that will read data from the specified stream
548   * and parses the LDIF records asynchronously using the specified number of
549   * threads.
550   *
551   * @param  inputStream  The input stream from which the data is to be read.
552   *                      It must not be {@code null}.
553   * @param  numParseThreads  If this value is greater than zero, then the
554   *                          specified number of threads will be used to
555   *                          asynchronously read and parse the LDIF file.
556   * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
557   *                         entries before they are returned.  This is normally
558   *                         {@code null}, which causes entries to be returned
559   *                         unaltered. This is particularly useful when parsing
560   *                         the input file in parallel because the entry
561   *                         translation is also done in parallel.
562   *
563   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
564   *      constructor for more details about asynchronous processing.
565   */
566  public LDIFReader(final InputStream inputStream, final int numParseThreads,
567                    final LDIFReaderEntryTranslator entryTranslator)
568  {
569    this(inputStream, numParseThreads, entryTranslator, null);
570  }
571
572
573
574  /**
575   * Creates a new LDIF reader that will read data from the specified stream
576   * and parses the LDIF records asynchronously using the specified number of
577   * threads.
578   *
579   * @param  inputStream             The input stream from which the data is to
580   *                                 be read.  It must not be {@code null}.
581   * @param  numParseThreads         If this value is greater than zero, then
582   *                                 the specified number of threads will be
583   *                                 used to asynchronously read and parse the
584   *                                 LDIF file.
585   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
586   *                                 entries before they are returned.  This is
587   *                                 normally {@code null}, which causes entries
588   *                                 to be returned unaltered.  This is
589   *                                 particularly useful when parsing the input
590   *                                 file in parallel because the entry
591   *                                 translation is also done in parallel.
592   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
593   *                                 apply to change records before they are
594   *                                 returned.  This is normally {@code null},
595   *                                 which causes change records to be returned
596   *                                 unaltered.  This is particularly useful
597   *                                 when parsing the input file in parallel
598   *                                 because the change record translation is
599   *                                 also done in parallel.
600   *
601   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
602   *      constructor for more details about asynchronous processing.
603   */
604  public LDIFReader(final InputStream inputStream, final int numParseThreads,
605              final LDIFReaderEntryTranslator entryTranslator,
606              final LDIFReaderChangeRecordTranslator changeRecordTranslator)
607  {
608    // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
609    this(inputStream, numParseThreads, entryTranslator, changeRecordTranslator,
610         "UTF-8");
611  }
612
613
614
615  /**
616   * Creates a new LDIF reader that will read data from the specified stream
617   * and parses the LDIF records asynchronously using the specified number of
618   * threads.
619   *
620   * @param  inputStream             The input stream from which the data is to
621   *                                 be read.  It must not be {@code null}.
622   * @param  numParseThreads         If this value is greater than zero, then
623   *                                 the specified number of threads will be
624   *                                 used to asynchronously read and parse the
625   *                                 LDIF file.
626   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
627   *                                 entries before they are returned.  This is
628   *                                 normally {@code null}, which causes entries
629   *                                 to be returned unaltered.  This is
630   *                                 particularly useful when parsing the input
631   *                                 file in parallel because the entry
632   *                                 translation is also done in parallel.
633   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
634   *                                 apply to change records before they are
635   *                                 returned.  This is normally {@code null},
636   *                                 which causes change records to be returned
637   *                                 unaltered.  This is particularly useful
638   *                                 when parsing the input file in parallel
639   *                                 because the change record translation is
640   *                                 also done in parallel.
641   * @param  characterSet            The character set to use when reading from
642   *                                 the input stream.  It must not be
643   *                                 {@code null}.
644   *
645   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
646   *      constructor for more details about asynchronous processing.
647   */
648  public LDIFReader(final InputStream inputStream, final int numParseThreads,
649              final LDIFReaderEntryTranslator entryTranslator,
650              final LDIFReaderChangeRecordTranslator changeRecordTranslator,
651              final String characterSet)
652  {
653    this(new BufferedReader(
654              new InputStreamReader(inputStream, Charset.forName(characterSet)),
655              DEFAULT_BUFFER_SIZE),
656         numParseThreads, entryTranslator, changeRecordTranslator);
657  }
658
659
660
661  /**
662   * Creates a new LDIF reader that will use the provided buffered reader to
663   * read the LDIF data.  The encoding of the underlying Reader must be set to
664   * "UTF-8" as required by RFC 2849.
665   *
666   * @param  reader  The buffered reader that will be used to read the LDIF
667   *                 data.  It must not be {@code null}.
668   */
669  public LDIFReader(final BufferedReader reader)
670  {
671    this(reader, 0);
672  }
673
674
675
676  /**
677   * Creates a new LDIF reader that will read data from the specified buffered
678   * reader and parses the LDIF records asynchronously using the specified
679   * number of threads.  The encoding of the underlying Reader must be set to
680   * "UTF-8" as required by RFC 2849.
681   *
682   * @param reader The buffered reader that will be used to read the LDIF data.
683   *               It must not be {@code null}.
684   * @param  numParseThreads  If this value is greater than zero, then the
685   *                          specified number of threads will be used to
686   *                          asynchronously read and parse the LDIF file.
687   *
688   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
689   *      constructor for more details about asynchronous processing.
690   */
691  public LDIFReader(final BufferedReader reader, final int numParseThreads)
692  {
693    this(reader, numParseThreads, null);
694  }
695
696
697
698  /**
699   * Creates a new LDIF reader that will read data from the specified buffered
700   * reader and parses the LDIF records asynchronously using the specified
701   * number of threads.  The encoding of the underlying Reader must be set to
702   * "UTF-8" as required by RFC 2849.
703   *
704   * @param reader The buffered reader that will be used to read the LDIF data.
705   *               It must not be {@code null}.
706   * @param  numParseThreads  If this value is greater than zero, then the
707   *                          specified number of threads will be used to
708   *                          asynchronously read and parse the LDIF file.
709   *                          This should only be set to greater than zero when
710   *                          performance analysis has demonstrated that reading
711   *                          and parsing the LDIF is a bottleneck.  The default
712   *                          synchronous processing is normally fast enough.
713   *                          There is little benefit in passing in a value
714   *                          greater than four (unless there is an
715   *                          LDIFReaderEntryTranslator that does time-consuming
716   *                          processing).  A value of zero implies the
717   *                          default behavior of reading and parsing LDIF
718   *                          records synchronously when one of the read
719   *                          methods is called.
720   * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
721   *                         entries before they are returned.  This is normally
722   *                         {@code null}, which causes entries to be returned
723   *                         unaltered. This is particularly useful when parsing
724   *                         the input file in parallel because the entry
725   *                         translation is also done in parallel.
726   */
727  public LDIFReader(final BufferedReader reader,
728                    final int numParseThreads,
729                    final LDIFReaderEntryTranslator entryTranslator)
730  {
731    this(reader, numParseThreads, entryTranslator, null);
732  }
733
734
735
736  /**
737   * Creates a new LDIF reader that will read data from the specified buffered
738   * reader and parses the LDIF records asynchronously using the specified
739   * number of threads.  The encoding of the underlying Reader must be set to
740   * "UTF-8" as required by RFC 2849.
741   *
742   * @param reader                   The buffered reader that will be used to
743   *                                 read the LDIF data.  It must not be
744   *                                 {@code null}.
745   * @param  numParseThreads         If this value is greater than zero, then
746   *                                 the specified number of threads will be
747   *                                 used to asynchronously read and parse the
748   *                                 LDIF file.
749   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
750   *                                 entries before they are returned.  This is
751   *                                 normally {@code null}, which causes entries
752   *                                 to be returned unaltered.  This is
753   *                                 particularly useful when parsing the input
754   *                                 file in parallel because the entry
755   *                                 translation is also done in parallel.
756   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
757   *                                 apply to change records before they are
758   *                                 returned.  This is normally {@code null},
759   *                                 which causes change records to be returned
760   *                                 unaltered.  This is particularly useful
761   *                                 when parsing the input file in parallel
762   *                                 because the change record translation is
763   *                                 also done in parallel.
764   */
765  public LDIFReader(final BufferedReader reader, final int numParseThreads,
766              final LDIFReaderEntryTranslator entryTranslator,
767              final LDIFReaderChangeRecordTranslator changeRecordTranslator)
768  {
769    Validator.ensureNotNull(reader);
770    Validator.ensureTrue(numParseThreads >= 0,
771               "LDIFReader.numParseThreads must not be negative.");
772
773    this.reader = reader;
774    this.entryTranslator = entryTranslator;
775    this.changeRecordTranslator = changeRecordTranslator;
776
777    duplicateValueBehavior = DuplicateValueBehavior.STRIP;
778    trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
779
780    relativeBasePath = DEFAULT_RELATIVE_BASE_PATH;
781
782    if (numParseThreads == 0)
783    {
784      isAsync = false;
785      asyncParser = null;
786      asyncParsingComplete = null;
787      asyncParsedRecords = null;
788    }
789    else
790    {
791      isAsync = true;
792      asyncParsingComplete = new AtomicBoolean(false);
793
794      // Decodes entries in parallel.
795      final LDAPSDKThreadFactory threadFactory =
796           new LDAPSDKThreadFactory("LDIFReader Worker", true, null);
797      final ParallelProcessor<UnparsedLDIFRecord, LDIFRecord> parallelParser =
798           new ParallelProcessor<>(
799                new RecordParser(), threadFactory, numParseThreads,
800                ASYNC_MIN_PER_PARSING_THREAD);
801
802      final BlockingQueue<UnparsedLDIFRecord> pendingQueue = new
803           ArrayBlockingQueue<>(ASYNC_QUEUE_SIZE);
804
805      // The output queue must be a little more than twice as big as the input
806      // queue to more easily handle being shutdown in the middle of processing
807      // when the queues are full and threads are blocked.
808      asyncParsedRecords = new ArrayBlockingQueue<>(2 * ASYNC_QUEUE_SIZE + 100);
809
810      asyncParser = new AsynchronousParallelProcessor<>(pendingQueue,
811           parallelParser, asyncParsedRecords);
812
813      final LineReaderThread lineReaderThread = new LineReaderThread();
814      lineReaderThread.start();
815    }
816  }
817
818
819
820  /**
821   * Reads entries from the LDIF file with the specified path and returns them
822   * as a {@code List}.  This is a convenience method that should only be used
823   * for data sets that are small enough so that running out of memory isn't a
824   * concern.
825   *
826   * @param  path  The path to the LDIF file containing the entries to be read.
827   *
828   * @return  A list of the entries read from the given LDIF file.
829   *
830   * @throws  IOException  If a problem occurs while attempting to read data
831   *                       from the specified file.
832   *
833   * @throws  LDIFException  If a problem is encountered while attempting to
834   *                         decode data read as LDIF.
835   */
836  public static List<Entry> readEntries(final String path)
837         throws IOException, LDIFException
838  {
839    return readEntries(new LDIFReader(path));
840  }
841
842
843
844  /**
845   * Reads entries from the specified LDIF file and returns them as a
846   * {@code List}.  This is a convenience method that should only be used for
847   * data sets that are small enough so that running out of memory isn't a
848   * concern.
849   *
850   * @param  file  A reference to the LDIF file containing the entries to be
851   *               read.
852   *
853   * @return  A list of the entries read from the given LDIF file.
854   *
855   * @throws  IOException  If a problem occurs while attempting to read data
856   *                       from the specified file.
857   *
858   * @throws  LDIFException  If a problem is encountered while attempting to
859   *                         decode data read as LDIF.
860   */
861  public static List<Entry> readEntries(final File file)
862         throws IOException, LDIFException
863  {
864    return readEntries(new LDIFReader(file));
865  }
866
867
868
869  /**
870   * Reads and decodes LDIF entries from the provided input stream and
871   * returns them as a {@code List}.  This is a convenience method that should
872   * only be used for data sets that are small enough so that running out of
873   * memory isn't a concern.
874   *
875   * @param  inputStream  The input stream from which the entries should be
876   *                      read.  The input stream will be closed before
877   *                      returning.
878   *
879   * @return  A list of the entries read from the given input stream.
880   *
881   * @throws  IOException  If a problem occurs while attempting to read data
882   *                       from the input stream.
883   *
884   * @throws  LDIFException  If a problem is encountered while attempting to
885   *                         decode data read as LDIF.
886   */
887  public static List<Entry> readEntries(final InputStream inputStream)
888         throws IOException, LDIFException
889  {
890    return readEntries(new LDIFReader(inputStream));
891  }
892
893
894
895  /**
896   * Reads entries from the provided LDIF reader and returns them as a list.
897   *
898   * @param  reader  The reader from which the entries should be read.  It will
899   *                 be closed before returning.
900   *
901   * @return  A list of the entries read from the provided reader.
902   *
903   * @throws  IOException  If a problem was encountered while attempting to read
904   *                       data from the LDIF data source.
905   *
906   * @throws  LDIFException  If a problem is encountered while attempting to
907   *                         decode data read as LDIF.
908   */
909  private static List<Entry> readEntries(final LDIFReader reader)
910          throws IOException, LDIFException
911  {
912    try
913    {
914      final ArrayList<Entry> entries = new ArrayList<>(10);
915      while (true)
916      {
917        final Entry e = reader.readEntry();
918        if (e == null)
919        {
920          break;
921        }
922
923        entries.add(e);
924      }
925
926      return entries;
927    }
928    finally
929    {
930      reader.close();
931    }
932  }
933
934
935
936  /**
937   * Closes this LDIF reader and the underlying LDIF source.
938   *
939   * @throws  IOException  If a problem occurs while closing the underlying LDIF
940   *                       source.
941   */
942  @Override()
943  public void close()
944         throws IOException
945  {
946    reader.close();
947
948    if (isAsync())
949    {
950      // Closing the reader will trigger the LineReaderThread to complete, but
951      // not if it's blocked submitting the next UnparsedLDIFRecord.  To avoid
952      // this, we clear out the completed output queue, which is larger than
953      // the input queue, so the LineReaderThread will stop reading and
954      // shutdown the asyncParser.
955      asyncParsedRecords.clear();
956    }
957  }
958
959
960
961  /**
962   * Indicates whether to ignore any duplicate values encountered while reading
963   * LDIF records.
964   *
965   * @return  {@code true} if duplicate values should be ignored, or
966   *          {@code false} if any LDIF records containing duplicate values
967   *          should be rejected.
968   *
969   * @deprecated  Use the {@link #getDuplicateValueBehavior} method instead.
970   */
971  @Deprecated()
972  public boolean ignoreDuplicateValues()
973  {
974    return (duplicateValueBehavior == DuplicateValueBehavior.STRIP);
975  }
976
977
978
979  /**
980   * Specifies whether to ignore any duplicate values encountered while reading
981   * LDIF records.
982   *
983   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
984   *                                attribute values encountered while reading
985   *                                LDIF records.
986   *
987   * @deprecated  Use the {@link #setDuplicateValueBehavior} method instead.
988   */
989  @Deprecated()
990  public void setIgnoreDuplicateValues(final boolean ignoreDuplicateValues)
991  {
992    if (ignoreDuplicateValues)
993    {
994      duplicateValueBehavior = DuplicateValueBehavior.STRIP;
995    }
996    else
997    {
998      duplicateValueBehavior = DuplicateValueBehavior.REJECT;
999    }
1000  }
1001
1002
1003
1004  /**
1005   * Retrieves the behavior that should be exhibited if the LDIF reader
1006   * encounters an entry with duplicate values.
1007   *
1008   * @return  The behavior that should be exhibited if the LDIF reader
1009   *          encounters an entry with duplicate values.
1010   */
1011  public DuplicateValueBehavior getDuplicateValueBehavior()
1012  {
1013    return duplicateValueBehavior;
1014  }
1015
1016
1017
1018  /**
1019   * Specifies the behavior that should be exhibited if the LDIF reader
1020   * encounters an entry with duplicate values.
1021   *
1022   * @param  duplicateValueBehavior  The behavior that should be exhibited if
1023   *                                 the LDIF reader encounters an entry with
1024   *                                 duplicate values.
1025   */
1026  public void setDuplicateValueBehavior(
1027                   final DuplicateValueBehavior duplicateValueBehavior)
1028  {
1029    this.duplicateValueBehavior = duplicateValueBehavior;
1030  }
1031
1032
1033
1034  /**
1035   * Indicates whether to strip off any illegal trailing spaces that may appear
1036   * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
1037   * specification strongly recommends that any value which legitimately
1038   * contains trailing spaces be base64-encoded, and any spaces which appear
1039   * after the end of non-base64-encoded values may therefore be considered
1040   * invalid.  If any such trailing spaces are encountered in an LDIF record and
1041   * they are not to be stripped, then an {@link LDIFException} will be thrown
1042   * for that record.
1043   * <BR><BR>
1044   * Note that this applies only to spaces after the end of a value, and not to
1045   * spaces which may appear at the end of a line for a value that is wrapped
1046   * and continued on the next line.
1047   *
1048   * @return  {@code true} if illegal trailing spaces should be stripped off, or
1049   *          {@code false} if LDIF records containing illegal trailing spaces
1050   *          should be rejected.
1051   *
1052   * @deprecated  Use the {@link #getTrailingSpaceBehavior} method instead.
1053   */
1054  @Deprecated()
1055  public boolean stripTrailingSpaces()
1056  {
1057    return (trailingSpaceBehavior == TrailingSpaceBehavior.STRIP);
1058  }
1059
1060
1061
1062  /**
1063   * Specifies whether to strip off any illegal trailing spaces that may appear
1064   * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
1065   * specification strongly recommends that any value which legitimately
1066   * contains trailing spaces be base64-encoded, and any spaces which appear
1067   * after the end of non-base64-encoded values may therefore be considered
1068   * invalid.  If any such trailing spaces are encountered in an LDIF record and
1069   * they are not to be stripped, then an {@link LDIFException} will be thrown
1070   * for that record.
1071   * <BR><BR>
1072   * Note that this applies only to spaces after the end of a value, and not to
1073   * spaces which may appear at the end of a line for a value that is wrapped
1074   * and continued on the next line.
1075   *
1076   * @param  stripTrailingSpaces  Indicates whether to strip off any illegal
1077   *                              trailing spaces, or {@code false} if LDIF
1078   *                              records containing them should be rejected.
1079   *
1080   * @deprecated  Use the {@link #setTrailingSpaceBehavior} method instead.
1081   */
1082  @Deprecated()
1083  public void setStripTrailingSpaces(final boolean stripTrailingSpaces)
1084  {
1085    trailingSpaceBehavior = stripTrailingSpaces
1086         ? TrailingSpaceBehavior.STRIP
1087         : TrailingSpaceBehavior.REJECT;
1088  }
1089
1090
1091
1092  /**
1093   * Retrieves the behavior that should be exhibited when encountering attribute
1094   * values which are not base64-encoded but contain trailing spaces.  The LDIF
1095   * specification strongly recommends that any value which legitimately
1096   * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
1097   * may be configured to automatically strip these spaces, to preserve them, or
1098   * to reject any entry or change record containing them.
1099   *
1100   * @return  The behavior that should be exhibited when encountering attribute
1101   *          values which are not base64-encoded but contain trailing spaces.
1102   */
1103  public TrailingSpaceBehavior getTrailingSpaceBehavior()
1104  {
1105    return trailingSpaceBehavior;
1106  }
1107
1108
1109
1110  /**
1111   * Specifies the behavior that should be exhibited when encountering attribute
1112   * values which are not base64-encoded but contain trailing spaces.  The LDIF
1113   * specification strongly recommends that any value which legitimately
1114   * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
1115   * may be configured to automatically strip these spaces, to preserve them, or
1116   * to reject any entry or change record containing them.
1117   *
1118   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
1119   *                                encountering attribute values which are not
1120   *                                base64-encoded but contain trailing spaces.
1121   */
1122  public void setTrailingSpaceBehavior(
1123                   final TrailingSpaceBehavior trailingSpaceBehavior)
1124  {
1125    this.trailingSpaceBehavior = trailingSpaceBehavior;
1126  }
1127
1128
1129
1130  /**
1131   * Retrieves the base path that will be prepended to relative paths in order
1132   * to obtain an absolute path.  This will only be used for "file:" URLs that
1133   * have paths which do not begin with a slash.
1134   *
1135   * @return  The base path that will be prepended to relative paths in order to
1136   *          obtain an absolute path.
1137   */
1138  public String getRelativeBasePath()
1139  {
1140    return relativeBasePath;
1141  }
1142
1143
1144
1145  /**
1146   * Specifies the base path that will be prepended to relative paths in order
1147   * to obtain an absolute path.  This will only be used for "file:" URLs that
1148   * have paths which do not begin with a space.
1149   *
1150   * @param  relativeBasePath  The base path that will be prepended to relative
1151   *                           paths in order to obtain an absolute path.
1152   */
1153  public void setRelativeBasePath(final String relativeBasePath)
1154  {
1155    setRelativeBasePath(new File(relativeBasePath));
1156  }
1157
1158
1159
1160  /**
1161   * Specifies the base path that will be prepended to relative paths in order
1162   * to obtain an absolute path.  This will only be used for "file:" URLs that
1163   * have paths which do not begin with a space.
1164   *
1165   * @param  relativeBasePath  The base path that will be prepended to relative
1166   *                           paths in order to obtain an absolute path.
1167   */
1168  public void setRelativeBasePath(final File relativeBasePath)
1169  {
1170    final String path = relativeBasePath.getAbsolutePath();
1171    if (path.endsWith(File.separator))
1172    {
1173      this.relativeBasePath = path;
1174    }
1175    else
1176    {
1177      this.relativeBasePath = path + File.separator;
1178    }
1179  }
1180
1181
1182
1183  /**
1184   * Retrieves the schema that will be used when reading LDIF records, if
1185   * defined.
1186   *
1187   * @return  The schema that will be used when reading LDIF records, or
1188   *          {@code null} if no schema should be used and all attributes should
1189   *          be treated as case-insensitive strings.
1190   */
1191  public Schema getSchema()
1192  {
1193    return schema;
1194  }
1195
1196
1197
1198  /**
1199   * Specifies the schema that should be used when reading LDIF records.
1200   *
1201   * @param  schema  The schema that should be used when reading LDIF records,
1202   *                 or {@code null} if no schema should be used and all
1203   *                 attributes should be treated as case-insensitive strings.
1204   */
1205  public void setSchema(final Schema schema)
1206  {
1207    this.schema = schema;
1208  }
1209
1210
1211
1212  /**
1213   * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1214   * change record.
1215   *
1216   * @return  The record read from the LDIF source, or {@code null} if there are
1217   *          no more entries to be read.
1218   *
1219   * @throws  IOException  If a problem occurs while trying to read from the
1220   *                       LDIF source.
1221   *
1222   * @throws  LDIFException  If the data read could not be parsed as an entry or
1223   *                         an LDIF change record.
1224   */
1225  public LDIFRecord readLDIFRecord()
1226         throws IOException, LDIFException
1227  {
1228    if (isAsync())
1229    {
1230      return readLDIFRecordAsync();
1231    }
1232    else
1233    {
1234      return readLDIFRecordInternal();
1235    }
1236  }
1237
1238
1239
1240  /**
1241   * Reads an entry from the LDIF source.
1242   *
1243   * @return  The entry read from the LDIF source, or {@code null} if there are
1244   *          no more entries to be read.
1245   *
1246   * @throws  IOException  If a problem occurs while attempting to read from the
1247   *                       LDIF source.
1248   *
1249   * @throws  LDIFException  If the data read could not be parsed as an entry.
1250   */
1251  public Entry readEntry()
1252         throws IOException, LDIFException
1253  {
1254    if (isAsync())
1255    {
1256      return readEntryAsync();
1257    }
1258    else
1259    {
1260      return readEntryInternal();
1261    }
1262  }
1263
1264
1265
1266  /**
1267   * Reads an LDIF change record from the LDIF source.  The LDIF record must
1268   * have a changetype.
1269   *
1270   * @return  The change record read from the LDIF source, or {@code null} if
1271   *          there are no more records to be read.
1272   *
1273   * @throws  IOException  If a problem occurs while attempting to read from the
1274   *                       LDIF source.
1275   *
1276   * @throws  LDIFException  If the data read could not be parsed as an LDIF
1277   *                         change record.
1278   */
1279  public LDIFChangeRecord readChangeRecord()
1280         throws IOException, LDIFException
1281  {
1282    return readChangeRecord(false);
1283  }
1284
1285
1286
1287  /**
1288   * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1289   * record does not have a changetype, then it may be assumed to be an add
1290   * change record.
1291   *
1292   * @param  defaultAdd  Indicates whether an LDIF record not containing a
1293   *                     changetype should be retrieved as an add change record.
1294   *                     If this is {@code false} and the record read does not
1295   *                     include a changetype, then an {@link LDIFException}
1296   *                     will be thrown.
1297   *
1298   * @return  The change record read from the LDIF source, or {@code null} if
1299   *          there are no more records to be read.
1300   *
1301   * @throws  IOException  If a problem occurs while attempting to read from the
1302   *                       LDIF source.
1303   *
1304   * @throws  LDIFException  If the data read could not be parsed as an LDIF
1305   *                         change record.
1306   */
1307  public LDIFChangeRecord readChangeRecord(final boolean defaultAdd)
1308         throws IOException, LDIFException
1309  {
1310    if (isAsync())
1311    {
1312      return readChangeRecordAsync(defaultAdd);
1313    }
1314    else
1315    {
1316      return readChangeRecordInternal(defaultAdd);
1317    }
1318  }
1319
1320
1321
1322  /**
1323   * Reads the next {@code LDIFRecord}, which was read and parsed by a different
1324   * thread.
1325   *
1326   * @return  The next parsed record or {@code null} if there are no more
1327   *          records to read.
1328   *
1329   * @throws IOException  If IOException was thrown when reading or parsing
1330   *                      the record.
1331   *
1332   * @throws LDIFException If LDIFException was thrown parsing the record.
1333   */
1334  private LDIFRecord readLDIFRecordAsync()
1335          throws IOException, LDIFException
1336  {
1337    Result<UnparsedLDIFRecord, LDIFRecord> result;
1338    LDIFRecord record = null;
1339    while (record == null)
1340    {
1341      result = readLDIFRecordResultAsync();
1342      if (result == null)
1343      {
1344        return null;
1345      }
1346
1347      record = result.getOutput();
1348
1349      // This is a special value that means we should skip this Entry.  We have
1350      // to use something different than null because null means EOF.
1351      if (record == SKIP_ENTRY)
1352      {
1353        record = null;
1354      }
1355    }
1356    return record;
1357  }
1358
1359
1360
1361  /**
1362   * Reads an entry asynchronously from the LDIF source.
1363   *
1364   * @return The entry read from the LDIF source, or {@code null} if there are
1365   *         no more entries to be read.
1366   *
1367   * @throws IOException   If a problem occurs while attempting to read from the
1368   *                       LDIF source.
1369   * @throws LDIFException If the data read could not be parsed as an entry.
1370   */
1371  private Entry readEntryAsync()
1372          throws IOException, LDIFException
1373  {
1374    Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1375    LDIFRecord record = null;
1376    while (record == null)
1377    {
1378      result = readLDIFRecordResultAsync();
1379      if (result == null)
1380      {
1381        return null;
1382      }
1383
1384      record = result.getOutput();
1385
1386      // This is a special value that means we should skip this Entry.  We have
1387      // to use something different than null because null means EOF.
1388      if (record == SKIP_ENTRY)
1389      {
1390        record = null;
1391      }
1392    }
1393
1394    if (record instanceof Entry)
1395    {
1396      return (Entry) record;
1397    }
1398    else if (record instanceof LDIFChangeRecord)
1399    {
1400      try
1401      {
1402        // Some LDIFChangeRecord can be converted to an Entry.  This is really
1403        // an edge case though.
1404        return ((LDIFChangeRecord)record).toEntry();
1405      }
1406      catch (final LDIFException e)
1407      {
1408        Debug.debugException(e);
1409        final long firstLineNumber = result.getInput().getFirstLineNumber();
1410        throw new LDIFException(e.getExceptionMessage(),
1411                                firstLineNumber, true, e);
1412      }
1413    }
1414
1415    throw new AssertionError("LDIFRecords must either be an Entry or an " +
1416                             "LDIFChangeRecord");
1417  }
1418
1419
1420
1421  /**
1422   * Reads an LDIF change record from the LDIF source asynchronously.
1423   * Optionally, if the LDIF record does not have a changetype, then it may be
1424   * assumed to be an add change record.
1425   *
1426   * @param defaultAdd Indicates whether an LDIF record not containing a
1427   *                   changetype should be retrieved as an add change record.
1428   *                   If this is {@code false} and the record read does not
1429   *                   include a changetype, then an {@link LDIFException} will
1430   *                   be thrown.
1431   *
1432   * @return The change record read from the LDIF source, or {@code null} if
1433   *         there are no more records to be read.
1434   *
1435   * @throws IOException   If a problem occurs while attempting to read from the
1436   *                       LDIF source.
1437   * @throws LDIFException If the data read could not be parsed as an LDIF
1438   *                       change record.
1439   */
1440  private LDIFChangeRecord readChangeRecordAsync(final boolean defaultAdd)
1441          throws IOException, LDIFException
1442  {
1443    Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1444    LDIFRecord record = null;
1445    while (record == null)
1446    {
1447      result = readLDIFRecordResultAsync();
1448      if (result == null)
1449      {
1450        return null;
1451      }
1452
1453      record = result.getOutput();
1454
1455      // This is a special value that means we should skip this Entry.  We have
1456      // to use something different than null because null means EOF.
1457      if (record == SKIP_ENTRY)
1458      {
1459        record = null;
1460      }
1461    }
1462
1463    if (record instanceof LDIFChangeRecord)
1464    {
1465      return (LDIFChangeRecord) record;
1466    }
1467    else if (record instanceof Entry)
1468    {
1469      if (defaultAdd)
1470      {
1471        return new LDIFAddChangeRecord((Entry) record);
1472      }
1473      else
1474      {
1475        final long firstLineNumber = result.getInput().getFirstLineNumber();
1476        throw new LDIFException(
1477             ERR_READ_NOT_CHANGE_RECORD.get(firstLineNumber), firstLineNumber,
1478             true);
1479      }
1480    }
1481
1482    throw new AssertionError("LDIFRecords must either be an Entry or an " +
1483                             "LDIFChangeRecord");
1484  }
1485
1486
1487
1488  /**
1489   * Reads the next LDIF record, which was read and parsed asynchronously by
1490   * separate threads.
1491   *
1492   * @return  The next LDIF record or {@code null} if there are no more records.
1493   *
1494   * @throws  IOException  If a problem occurs while attempting to read from the
1495   *                       LDIF source.
1496   *
1497   * @throws  LDIFException  If the data read could not be parsed as an entry.
1498   */
1499  private Result<UnparsedLDIFRecord, LDIFRecord> readLDIFRecordResultAsync()
1500          throws IOException, LDIFException
1501  {
1502    Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1503
1504    // If the asynchronous reading and parsing is complete, then we don't have
1505    // to block waiting for the next record to show up on the queue.  If there
1506    // isn't a record there, then return null (EOF) right away.
1507    if (asyncParsingComplete.get())
1508    {
1509      result = asyncParsedRecords.poll();
1510    }
1511    else
1512    {
1513      try
1514      {
1515        // We probably could just do a asyncParsedRecords.take() here, but
1516        // there are some edge case error scenarios where
1517        // asyncParsingComplete might be set without a special EOF sentinel
1518        // Result enqueued.  So to guard against this, we have a very cautious
1519        // polling interval of 1 second.  During normal processing, we never
1520        // have to wait for this to expire, when there is something to do
1521        // (like shutdown).
1522        while ((result == null) && (!asyncParsingComplete.get()))
1523        {
1524          result = asyncParsedRecords.poll(1, TimeUnit.SECONDS);
1525        }
1526
1527        // There's a very small chance that we missed the value, so double-check
1528        if (result == null)
1529        {
1530          result = asyncParsedRecords.poll();
1531        }
1532      }
1533      catch (final InterruptedException e)
1534      {
1535        Debug.debugException(e);
1536        Thread.currentThread().interrupt();
1537        throw new IOException(e);
1538      }
1539    }
1540    if (result == null)
1541    {
1542      return null;
1543    }
1544
1545    rethrow(result.getFailureCause());
1546
1547    // Check if we reached the end of the input
1548    final UnparsedLDIFRecord unparsedRecord = result.getInput();
1549    if (unparsedRecord.isEOF())
1550    {
1551      // This might have been set already by the LineReaderThread, but
1552      // just in case it hasn't gotten to it yet, do so here.
1553      asyncParsingComplete.set(true);
1554
1555      // Enqueue this EOF result again for any other thread that might be
1556      // blocked in asyncParsedRecords.take() even though having multiple
1557      // threads call this method concurrently breaks the contract of this
1558      // class.
1559      try
1560      {
1561        asyncParsedRecords.put(result);
1562      }
1563      catch (final InterruptedException e)
1564      {
1565        // We shouldn't ever get interrupted because the put won't ever block.
1566        // Once we are done reading, this is the only item left in the queue,
1567        // so we should always be able to re-enqueue it.
1568        Debug.debugException(e);
1569        Thread.currentThread().interrupt();
1570      }
1571      return null;
1572    }
1573
1574    return result;
1575  }
1576
1577
1578
1579  /**
1580   * Indicates whether this LDIF reader was constructed to perform asynchronous
1581   * processing.
1582   *
1583   * @return  {@code true} if this LDIFReader was constructed to perform
1584   *          asynchronous processing, or {@code false} if not.
1585   */
1586  private boolean isAsync()
1587  {
1588    return isAsync;
1589  }
1590
1591
1592
1593  /**
1594   * If not {@code null}, rethrows the specified Throwable as either an
1595   * IOException or LDIFException.
1596   *
1597   * @param t  The exception to rethrow.  If it's {@code null}, then nothing
1598   *           is thrown.
1599   *
1600   * @throws IOException   If t is an IOException or a checked Exception that
1601   *                       is not an LDIFException.
1602   * @throws LDIFException  If t is an LDIFException.
1603   */
1604  static void rethrow(final Throwable t)
1605         throws IOException, LDIFException
1606  {
1607    if (t == null)
1608    {
1609      return;
1610    }
1611
1612    if (t instanceof IOException)
1613    {
1614      throw (IOException) t;
1615    }
1616    else if (t instanceof LDIFException)
1617    {
1618      throw (LDIFException) t;
1619    }
1620    else if (t instanceof RuntimeException)
1621    {
1622      throw (RuntimeException) t;
1623    }
1624    else if (t instanceof Error)
1625    {
1626      throw (Error) t;
1627    }
1628    else
1629    {
1630      throw new IOException(t);
1631    }
1632  }
1633
1634
1635
1636  /**
1637   * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1638   * change record.
1639   *
1640   * @return The record read from the LDIF source, or {@code null} if there are
1641   *         no more entries to be read.
1642   *
1643   * @throws IOException   If a problem occurs while trying to read from the
1644   *                       LDIF source.
1645   * @throws LDIFException If the data read could not be parsed as an entry or
1646   *                       an LDIF change record.
1647   */
1648  private LDIFRecord readLDIFRecordInternal()
1649       throws IOException, LDIFException
1650  {
1651    final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1652    return decodeRecord(unparsedRecord, relativeBasePath, schema);
1653  }
1654
1655
1656
1657  /**
1658   * Reads an entry from the LDIF source.
1659   *
1660   * @return The entry read from the LDIF source, or {@code null} if there are
1661   *         no more entries to be read.
1662   *
1663   * @throws IOException   If a problem occurs while attempting to read from the
1664   *                       LDIF source.
1665   * @throws LDIFException If the data read could not be parsed as an entry.
1666   */
1667  private Entry readEntryInternal()
1668       throws IOException, LDIFException
1669  {
1670    Entry e = null;
1671    while (e == null)
1672    {
1673      final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1674      if (unparsedRecord.isEOF())
1675      {
1676        return null;
1677      }
1678
1679      e = decodeEntry(unparsedRecord, relativeBasePath);
1680      Debug.debugLDIFRead(e);
1681
1682      if (entryTranslator != null)
1683      {
1684        e = entryTranslator.translate(e, unparsedRecord.getFirstLineNumber());
1685      }
1686    }
1687    return e;
1688  }
1689
1690
1691
1692  /**
1693   * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1694   * record does not have a changetype, then it may be assumed to be an add
1695   * change record.
1696   *
1697   * @param defaultAdd Indicates whether an LDIF record not containing a
1698   *                   changetype should be retrieved as an add change record.
1699   *                   If this is {@code false} and the record read does not
1700   *                   include a changetype, then an {@link LDIFException} will
1701   *                   be thrown.
1702   *
1703   * @return The change record read from the LDIF source, or {@code null} if
1704   *         there are no more records to be read.
1705   *
1706   * @throws IOException   If a problem occurs while attempting to read from the
1707   *                       LDIF source.
1708   * @throws LDIFException If the data read could not be parsed as an LDIF
1709   *                       change record.
1710   */
1711  private LDIFChangeRecord readChangeRecordInternal(final boolean defaultAdd)
1712       throws IOException, LDIFException
1713  {
1714    LDIFChangeRecord r = null;
1715    while (r == null)
1716    {
1717      final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1718      if (unparsedRecord.isEOF())
1719      {
1720        return null;
1721      }
1722
1723      r = decodeChangeRecord(unparsedRecord, relativeBasePath, defaultAdd,
1724           schema);
1725      Debug.debugLDIFRead(r);
1726
1727      if (changeRecordTranslator != null)
1728      {
1729        r = changeRecordTranslator.translate(r,
1730             unparsedRecord.getFirstLineNumber());
1731      }
1732    }
1733    return r;
1734  }
1735
1736
1737
1738  /**
1739   * Reads a record (either an entry or a change record) from the LDIF source
1740   * and places it in the line list.
1741   *
1742   * @return  The line number for the first line of the entry that was read.
1743   *
1744   * @throws  IOException  If a problem occurs while attempting to read from the
1745   *                       LDIF source.
1746   *
1747   * @throws  LDIFException  If the data read could not be parsed as a valid
1748   *                         LDIF record.
1749   */
1750  private UnparsedLDIFRecord readUnparsedRecord()
1751         throws IOException, LDIFException
1752  {
1753    final ArrayList<StringBuilder> lineList = new ArrayList<>(20);
1754    boolean lastWasComment = false;
1755    long firstLineNumber = lineNumberCounter + 1;
1756    while (true)
1757    {
1758      final String line = reader.readLine();
1759      lineNumberCounter++;
1760
1761      if (line == null)
1762      {
1763        // We've hit the end of the LDIF source.  If we haven't read any entry
1764        // data, then return null.  Otherwise, the last entry wasn't followed by
1765        // a blank line, which is OK, and we should decode that entry.
1766        if (lineList.isEmpty())
1767        {
1768          return new UnparsedLDIFRecord(new ArrayList<StringBuilder>(0),
1769               duplicateValueBehavior, trailingSpaceBehavior, schema, -1);
1770        }
1771        else
1772        {
1773          break;
1774        }
1775      }
1776
1777      if (line.isEmpty())
1778      {
1779        // It's a blank line.  If we have read entry data, then this signals the
1780        // end of the entry.  Otherwise, it's an extra space between entries,
1781        // which is OK.
1782        lastWasComment = false;
1783        if (lineList.isEmpty())
1784        {
1785          firstLineNumber++;
1786          continue;
1787        }
1788        else
1789        {
1790          break;
1791        }
1792      }
1793
1794      if (line.charAt(0) == ' ')
1795      {
1796        // The line starts with a space, which means that it must be a
1797        // continuation of the previous line.  This is true even if the last
1798        // line was a comment.
1799        if (lastWasComment)
1800        {
1801          // What we've read is part of a comment, so we don't care about its
1802          // content.
1803        }
1804        else if (lineList.isEmpty())
1805        {
1806          throw new LDIFException(
1807                         ERR_READ_UNEXPECTED_FIRST_SPACE.get(lineNumberCounter),
1808                         lineNumberCounter, false);
1809        }
1810        else
1811        {
1812          lineList.get(lineList.size() - 1).append(line.substring(1));
1813          lastWasComment = false;
1814        }
1815      }
1816      else if (line.charAt(0) == '#')
1817      {
1818        lastWasComment = true;
1819      }
1820      else
1821      {
1822        // We want to make sure that we skip over the "version:" line if it
1823        // exists, but that should only occur at the beginning of an entry where
1824        // it can't be confused with a possible "version" attribute.
1825        if (lineList.isEmpty() && line.startsWith("version:"))
1826        {
1827          lastWasComment = true;
1828        }
1829        else
1830        {
1831          lineList.add(new StringBuilder(line));
1832          lastWasComment = false;
1833        }
1834      }
1835    }
1836
1837    return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1838         trailingSpaceBehavior, schema, firstLineNumber);
1839  }
1840
1841
1842
1843  /**
1844   * Decodes the provided set of LDIF lines as an entry.  The provided set of
1845   * lines must contain exactly one entry.  Long lines may be wrapped as per the
1846   * LDIF specification, and it is acceptable to have one or more blank lines
1847   * following the entry. A default trailing space behavior of
1848   * {@link TrailingSpaceBehavior#REJECT} will be used.
1849   *
1850   * @param  ldifLines  The set of lines that comprise the LDIF representation
1851   *                    of the entry.  It must not be {@code null} or empty.
1852   *
1853   * @return  The entry read from LDIF.
1854   *
1855   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1856   *                         entry.
1857   */
1858  public static Entry decodeEntry(final String... ldifLines)
1859         throws LDIFException
1860  {
1861    final Entry e = decodeEntry(prepareRecord(DuplicateValueBehavior.STRIP,
1862         TrailingSpaceBehavior.REJECT, null, ldifLines),
1863         DEFAULT_RELATIVE_BASE_PATH);
1864    Debug.debugLDIFRead(e);
1865    return e;
1866  }
1867
1868
1869
1870  /**
1871   * Decodes the provided set of LDIF lines as an entry.  The provided set of
1872   * lines must contain exactly one entry.  Long lines may be wrapped as per the
1873   * LDIF specification, and it is acceptable to have one or more blank lines
1874   * following the entry. A default trailing space behavior of
1875   * {@link TrailingSpaceBehavior#REJECT} will be used.
1876   *
1877   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1878   *                                attribute values encountered while parsing.
1879   * @param  schema                 The schema to use when parsing the record,
1880   *                                if applicable.
1881   * @param  ldifLines              The set of lines that comprise the LDIF
1882   *                                representation of the entry.  It must not be
1883   *                                {@code null} or empty.
1884   *
1885   * @return  The entry read from LDIF.
1886   *
1887   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1888   *                         entry.
1889   */
1890  public static Entry decodeEntry(final boolean ignoreDuplicateValues,
1891                                  final Schema schema,
1892                                  final String... ldifLines)
1893         throws LDIFException
1894  {
1895    return decodeEntry(ignoreDuplicateValues, TrailingSpaceBehavior.REJECT,
1896         schema, ldifLines);
1897  }
1898
1899
1900
1901  /**
1902   * Decodes the provided set of LDIF lines as an entry.  The provided set of
1903   * lines must contain exactly one entry.  Long lines may be wrapped as per the
1904   * LDIF specification, and it is acceptable to have one or more blank lines
1905   * following the entry.
1906   *
1907   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1908   *                                attribute values encountered while parsing.
1909   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
1910   *                                encountering attribute values which are not
1911   *                                base64-encoded but contain trailing spaces.
1912   *                                It must not be {@code null}.
1913   * @param  schema                 The schema to use when parsing the record,
1914   *                                if applicable.
1915   * @param  ldifLines              The set of lines that comprise the LDIF
1916   *                                representation of the entry.  It must not be
1917   *                                {@code null} or empty.
1918   *
1919   * @return  The entry read from LDIF.
1920   *
1921   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1922   *                         entry.
1923   */
1924  public static Entry decodeEntry(
1925         final boolean ignoreDuplicateValues,
1926         final TrailingSpaceBehavior trailingSpaceBehavior,
1927         final Schema schema,
1928         final String... ldifLines) throws LDIFException
1929  {
1930    final Entry e = decodeEntry(prepareRecord(
1931              (ignoreDuplicateValues
1932                   ? DuplicateValueBehavior.STRIP
1933                   : DuplicateValueBehavior.REJECT),
1934         trailingSpaceBehavior, schema, ldifLines),
1935         DEFAULT_RELATIVE_BASE_PATH);
1936    Debug.debugLDIFRead(e);
1937    return e;
1938  }
1939
1940
1941
1942  /**
1943   * Decodes the provided set of LDIF lines as an LDIF change record.  The
1944   * provided set of lines must contain exactly one change record and it must
1945   * include a changetype.  Long lines may be wrapped as per the LDIF
1946   * specification, and it is acceptable to have one or more blank lines
1947   * following the entry.
1948   *
1949   * @param  ldifLines  The set of lines that comprise the LDIF representation
1950   *                    of the change record.  It must not be {@code null} or
1951   *                    empty.
1952   *
1953   * @return  The change record read from LDIF.
1954   *
1955   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1956   *                         change record.
1957   */
1958  public static LDIFChangeRecord decodeChangeRecord(final String... ldifLines)
1959         throws LDIFException
1960  {
1961    return decodeChangeRecord(false, ldifLines);
1962  }
1963
1964
1965
1966  /**
1967   * Decodes the provided set of LDIF lines as an LDIF change record.  The
1968   * provided set of lines must contain exactly one change record.  Long lines
1969   * may be wrapped as per the LDIF specification, and it is acceptable to have
1970   * one or more blank lines following the entry.
1971   *
1972   * @param  defaultAdd  Indicates whether an LDIF record not containing a
1973   *                     changetype should be retrieved as an add change record.
1974   *                     If this is {@code false} and the record read does not
1975   *                     include a changetype, then an {@link LDIFException}
1976   *                     will be thrown.
1977   * @param  ldifLines  The set of lines that comprise the LDIF representation
1978   *                    of the change record.  It must not be {@code null} or
1979   *                    empty.
1980   *
1981   * @return  The change record read from LDIF.
1982   *
1983   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1984   *                         change record.
1985   */
1986  public static LDIFChangeRecord decodeChangeRecord(final boolean defaultAdd,
1987                                                    final String... ldifLines)
1988         throws LDIFException
1989  {
1990    final LDIFChangeRecord r =
1991         decodeChangeRecord(
1992              prepareRecord(DuplicateValueBehavior.STRIP,
1993                   TrailingSpaceBehavior.REJECT, null, ldifLines),
1994              DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null);
1995    Debug.debugLDIFRead(r);
1996    return r;
1997  }
1998
1999
2000
2001  /**
2002   * Decodes the provided set of LDIF lines as an LDIF change record.  The
2003   * provided set of lines must contain exactly one change record.  Long lines
2004   * may be wrapped as per the LDIF specification, and it is acceptable to have
2005   * one or more blank lines following the entry.
2006   *
2007   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
2008   *                                attribute values encountered while parsing.
2009   * @param  schema                 The schema to use when processing the change
2010   *                                record, or {@code null} if no schema should
2011   *                                be used and all values should be treated as
2012   *                                case-insensitive strings.
2013   * @param  defaultAdd             Indicates whether an LDIF record not
2014   *                                containing a changetype should be retrieved
2015   *                                as an add change record.  If this is
2016   *                                {@code false} and the record read does not
2017   *                                include a changetype, then an
2018   *                                {@link LDIFException} will be thrown.
2019   * @param  ldifLines              The set of lines that comprise the LDIF
2020   *                                representation of the change record.  It
2021   *                                must not be {@code null} or empty.
2022   *
2023   * @return  The change record read from LDIF.
2024   *
2025   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2026   *                         change record.
2027   */
2028  public static LDIFChangeRecord decodeChangeRecord(
2029                                      final boolean ignoreDuplicateValues,
2030                                      final Schema schema,
2031                                      final boolean defaultAdd,
2032                                      final String... ldifLines)
2033         throws LDIFException
2034  {
2035    return decodeChangeRecord(ignoreDuplicateValues,
2036         TrailingSpaceBehavior.REJECT, schema, defaultAdd, ldifLines);
2037  }
2038
2039
2040
2041  /**
2042   * Decodes the provided set of LDIF lines as an LDIF change record.  The
2043   * provided set of lines must contain exactly one change record.  Long lines
2044   * may be wrapped as per the LDIF specification, and it is acceptable to have
2045   * one or more blank lines following the entry.
2046   *
2047   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
2048   *                                attribute values encountered while parsing.
2049   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
2050   *                                encountering attribute values which are not
2051   *                                base64-encoded but contain trailing spaces.
2052   *                                It must not be {@code null}.
2053   * @param  schema                 The schema to use when processing the change
2054   *                                record, or {@code null} if no schema should
2055   *                                be used and all values should be treated as
2056   *                                case-insensitive strings.
2057   * @param  defaultAdd             Indicates whether an LDIF record not
2058   *                                containing a changetype should be retrieved
2059   *                                as an add change record.  If this is
2060   *                                {@code false} and the record read does not
2061   *                                include a changetype, then an
2062   *                                {@link LDIFException} will be thrown.
2063   * @param  ldifLines              The set of lines that comprise the LDIF
2064   *                                representation of the change record.  It
2065   *                                must not be {@code null} or empty.
2066   *
2067   * @return  The change record read from LDIF.
2068   *
2069   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2070   *                         change record.
2071   */
2072  public static LDIFChangeRecord decodeChangeRecord(
2073                     final boolean ignoreDuplicateValues,
2074                     final TrailingSpaceBehavior trailingSpaceBehavior,
2075                     final Schema schema,
2076                     final boolean defaultAdd,
2077                     final String... ldifLines)
2078         throws LDIFException
2079  {
2080    final LDIFChangeRecord r = decodeChangeRecord(
2081         prepareRecord(
2082              (ignoreDuplicateValues
2083                   ? DuplicateValueBehavior.STRIP
2084                   : DuplicateValueBehavior.REJECT),
2085              trailingSpaceBehavior, schema, ldifLines),
2086         DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null);
2087    Debug.debugLDIFRead(r);
2088    return r;
2089  }
2090
2091
2092
2093  /**
2094   * Parses the provided set of lines into a list of {@code StringBuilder}
2095   * objects suitable for decoding into an entry or LDIF change record.
2096   * Comments will be ignored and wrapped lines will be unwrapped.
2097   *
2098   * @param  duplicateValueBehavior  The behavior that should be exhibited if
2099   *                                 the LDIF reader encounters an entry with
2100   *                                 duplicate values.
2101   * @param  trailingSpaceBehavior   The behavior that should be exhibited when
2102   *                                 encountering attribute values which are not
2103   *                                 base64-encoded but contain trailing spaces.
2104   * @param  schema                  The schema to use when parsing the record,
2105   *                                 if applicable.
2106   * @param  ldifLines               The set of lines that comprise the record
2107   *                                 to decode.  It must not be {@code null} or
2108   *                                 empty.
2109   *
2110   * @return  The prepared list of {@code StringBuilder} objects ready to be
2111   *          decoded.
2112   *
2113   * @throws  LDIFException  If the provided lines do not contain valid LDIF
2114   *                         content.
2115   */
2116  private static UnparsedLDIFRecord prepareRecord(
2117                      final DuplicateValueBehavior duplicateValueBehavior,
2118                      final TrailingSpaceBehavior trailingSpaceBehavior,
2119                      final Schema schema, final String... ldifLines)
2120          throws LDIFException
2121  {
2122    Validator.ensureNotNull(ldifLines);
2123    Validator.ensureFalse(ldifLines.length == 0,
2124         "LDIFReader.prepareRecord.ldifLines must not be empty.");
2125
2126    boolean lastWasComment = false;
2127    final ArrayList<StringBuilder> lineList = new ArrayList<>(ldifLines.length);
2128    for (int i=0; i < ldifLines.length; i++)
2129    {
2130      final String line = ldifLines[i];
2131      if (line.isEmpty())
2132      {
2133        // This is only acceptable if there are no more non-empty lines in the
2134        // array.
2135        for (int j=i+1; j < ldifLines.length; j++)
2136        {
2137          if (! ldifLines[j].isEmpty())
2138          {
2139            throw new LDIFException(ERR_READ_UNEXPECTED_BLANK.get(i), i, true,
2140                                    ldifLines, null);
2141          }
2142
2143          // If we've gotten here, then we know that we're at the end of the
2144          // entry.  If we have read data, then we can decode it as an entry.
2145          // Otherwise, there was no real data in the provided LDIF lines.
2146          if (lineList.isEmpty())
2147          {
2148            throw new LDIFException(ERR_READ_ONLY_BLANKS.get(), 0, true,
2149                                    ldifLines, null);
2150          }
2151          else
2152          {
2153            return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2154                 trailingSpaceBehavior, schema, 0);
2155          }
2156        }
2157      }
2158
2159      if (line.charAt(0) == ' ')
2160      {
2161        if (i > 0)
2162        {
2163          if (! lastWasComment)
2164          {
2165            lineList.get(lineList.size() - 1).append(line.substring(1));
2166          }
2167        }
2168        else
2169        {
2170          throw new LDIFException(
2171                         ERR_READ_UNEXPECTED_FIRST_SPACE_NO_NUMBER.get(), 0,
2172                         true, ldifLines, null);
2173        }
2174      }
2175      else if (line.charAt(0) == '#')
2176      {
2177        lastWasComment = true;
2178      }
2179      else
2180      {
2181        lineList.add(new StringBuilder(line));
2182        lastWasComment = false;
2183      }
2184    }
2185
2186    if (lineList.isEmpty())
2187    {
2188      throw new LDIFException(ERR_READ_NO_DATA.get(), 0, true, ldifLines, null);
2189    }
2190    else
2191    {
2192      return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2193           trailingSpaceBehavior, schema, 0);
2194    }
2195  }
2196
2197
2198
2199  /**
2200   * Decodes the unparsed record that was read from the LDIF source.  It may be
2201   * either an entry or an LDIF change record.
2202   *
2203   * @param  unparsedRecord    The unparsed LDIF record that was read from the
2204   *                           input.  It must not be {@code null} or empty.
2205   * @param  relativeBasePath  The base path that will be prepended to relative
2206   *                           paths in order to obtain an absolute path.
2207   * @param  schema            The schema to use when parsing.
2208   *
2209   * @return  The parsed record, or {@code null} if there are no more entries to
2210   *          be read.
2211   *
2212   * @throws  LDIFException  If the data read could not be parsed as an entry or
2213   *                         an LDIF change record.
2214   */
2215  private static LDIFRecord decodeRecord(
2216                                 final UnparsedLDIFRecord unparsedRecord,
2217                                 final String relativeBasePath,
2218                                 final Schema schema)
2219       throws LDIFException
2220  {
2221    // If there was an error reading from the input, then we rethrow it here.
2222    final Exception readError = unparsedRecord.getFailureCause();
2223    if (readError != null)
2224    {
2225      if (readError instanceof LDIFException)
2226      {
2227        // If the error was an LDIFException, which will normally be the case,
2228        // then rethrow it with all of the same state.  We could just
2229        //   throw (LDIFException) readError;
2230        // but that's considered bad form.
2231        final LDIFException ldifEx = (LDIFException) readError;
2232        throw new LDIFException(ldifEx.getMessage(),
2233                                ldifEx.getLineNumber(),
2234                                ldifEx.mayContinueReading(),
2235                                ldifEx.getDataLines(),
2236                                ldifEx.getCause());
2237      }
2238      else
2239      {
2240        throw new LDIFException(StaticUtils.getExceptionMessage(readError),
2241             -1, true, readError);
2242      }
2243    }
2244
2245    if (unparsedRecord.isEOF())
2246    {
2247      return null;
2248    }
2249
2250    final ArrayList<StringBuilder> lineList = unparsedRecord.getLineList();
2251    if (unparsedRecord.getLineList() == null)
2252    {
2253      return null;  // We can get here if there was an error reading the lines.
2254    }
2255
2256    final LDIFRecord r;
2257    if (lineList.size() == 1)
2258    {
2259      r = decodeEntry(unparsedRecord, relativeBasePath);
2260    }
2261    else
2262    {
2263      final String lowerSecondLine =
2264           StaticUtils.toLowerCase(lineList.get(1).toString());
2265      if (lowerSecondLine.startsWith("control:") ||
2266          lowerSecondLine.startsWith("changetype:"))
2267      {
2268        r = decodeChangeRecord(unparsedRecord, relativeBasePath, true, schema);
2269      }
2270      else
2271      {
2272        r = decodeEntry(unparsedRecord, relativeBasePath);
2273      }
2274    }
2275
2276    Debug.debugLDIFRead(r);
2277    return r;
2278  }
2279
2280
2281
2282  /**
2283   * Decodes the provided set of LDIF lines as an entry.  The provided list must
2284   * not contain any blank lines or comments, and lines are not allowed to be
2285   * wrapped.
2286   *
2287   * @param  unparsedRecord   The unparsed LDIF record that was read from the
2288   *                          input.  It must not be {@code null} or empty.
2289   * @param  relativeBasePath  The base path that will be prepended to relative
2290   *                           paths in order to obtain an absolute path.
2291   *
2292   * @return  The entry read from LDIF.
2293   *
2294   * @throws  LDIFException  If the provided LDIF data cannot be read as an
2295   *                         entry.
2296   */
2297  private static Entry decodeEntry(final UnparsedLDIFRecord unparsedRecord,
2298                                   final String relativeBasePath)
2299          throws LDIFException
2300  {
2301    final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2302    final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2303
2304    final Iterator<StringBuilder> iterator = ldifLines.iterator();
2305
2306    // The first line must start with either "version:" or "dn:".  If the first
2307    // line starts with "version:" then the second must start with "dn:".
2308    StringBuilder line = iterator.next();
2309    handleTrailingSpaces(line, null, firstLineNumber,
2310         unparsedRecord.getTrailingSpaceBehavior());
2311    int colonPos = line.indexOf(":");
2312    if ((colonPos > 0) &&
2313        line.substring(0, colonPos).equalsIgnoreCase("version"))
2314    {
2315      // The first line is "version:".  Under most conditions, this will be
2316      // handled by the LDIF reader, but this can happen if you call
2317      // decodeEntry with a set of data that includes a version.  At any rate,
2318      // read the next line, which must specify the DN.
2319      line = iterator.next();
2320      handleTrailingSpaces(line, null, firstLineNumber,
2321           unparsedRecord.getTrailingSpaceBehavior());
2322    }
2323
2324    colonPos = line.indexOf(":");
2325    if ((colonPos < 0) ||
2326         (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2327    {
2328      throw new LDIFException(
2329           ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2330           firstLineNumber, true, ldifLines, null);
2331    }
2332
2333    final String dn;
2334    final int length = line.length();
2335    if (length == (colonPos+1))
2336    {
2337      // The colon was the last character on the line.  This is acceptable and
2338      // indicates that the entry has the null DN.
2339      dn = "";
2340    }
2341    else if (line.charAt(colonPos+1) == ':')
2342    {
2343      // Skip over any spaces leading up to the value, and then the rest of the
2344      // string is the base64-encoded DN.
2345      int pos = colonPos+2;
2346      while ((pos < length) && (line.charAt(pos) == ' '))
2347      {
2348        pos++;
2349      }
2350
2351      try
2352      {
2353        final byte[] dnBytes = Base64.decode(line.substring(pos));
2354        dn = StaticUtils.toUTF8String(dnBytes);
2355      }
2356      catch (final ParseException pe)
2357      {
2358        Debug.debugException(pe);
2359        throw new LDIFException(
2360                       ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2361                                                            pe.getMessage()),
2362                       firstLineNumber, true, ldifLines, pe);
2363      }
2364      catch (final Exception e)
2365      {
2366        Debug.debugException(e);
2367        throw new LDIFException(
2368                       ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, e),
2369                       firstLineNumber, true, ldifLines, e);
2370      }
2371    }
2372    else
2373    {
2374      // Skip over any spaces leading up to the value, and then the rest of the
2375      // string is the DN.
2376      int pos = colonPos+1;
2377      while ((pos < length) && (line.charAt(pos) == ' '))
2378      {
2379        pos++;
2380      }
2381
2382      dn = line.substring(pos);
2383    }
2384
2385
2386    // The remaining lines must be the attributes for the entry.  However, we
2387    // will allow the case in which an entry does not have any attributes, to be
2388    // able to support reading search result entries in which no attributes were
2389    // returned.
2390    if (! iterator.hasNext())
2391    {
2392      return new Entry(dn, unparsedRecord.getSchema());
2393    }
2394
2395    return new Entry(dn, unparsedRecord.getSchema(),
2396         parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2397              unparsedRecord.getTrailingSpaceBehavior(),
2398              unparsedRecord.getSchema(), ldifLines, iterator, relativeBasePath,
2399              firstLineNumber));
2400  }
2401
2402
2403
2404  /**
2405   * Decodes the provided set of LDIF lines as a change record.  The provided
2406   * list must not contain any blank lines or comments, and lines are not
2407   * allowed to be wrapped.
2408   *
2409   * @param  unparsedRecord    The unparsed LDIF record that was read from the
2410   *                           input.  It must not be {@code null} or empty.
2411   * @param  relativeBasePath  The base path that will be prepended to relative
2412   *                           paths in order to obtain an absolute path.
2413   * @param  defaultAdd        Indicates whether an LDIF record not containing a
2414   *                           changetype should be retrieved as an add change
2415   *                           record.  If this is {@code false} and the record
2416   *                           read does not include a changetype, then an
2417   *                           {@link LDIFException} will be thrown.
2418   * @param  schema            The schema to use in parsing.
2419   *
2420   * @return  The change record read from LDIF.
2421   *
2422   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2423   *                         change record.
2424   */
2425  private static LDIFChangeRecord decodeChangeRecord(
2426                                       final UnparsedLDIFRecord unparsedRecord,
2427                                       final String relativeBasePath,
2428                                       final boolean defaultAdd,
2429                                       final Schema schema)
2430          throws LDIFException
2431  {
2432    final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2433    final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2434
2435    Iterator<StringBuilder> iterator = ldifLines.iterator();
2436
2437    // The first line must start with either "version:" or "dn:".  If the first
2438    // line starts with "version:" then the second must start with "dn:".
2439    StringBuilder line = iterator.next();
2440    handleTrailingSpaces(line, null, firstLineNumber,
2441         unparsedRecord.getTrailingSpaceBehavior());
2442    int colonPos = line.indexOf(":");
2443    int linesRead = 1;
2444    if ((colonPos > 0) &&
2445        line.substring(0, colonPos).equalsIgnoreCase("version"))
2446    {
2447      // The first line is "version:".  Under most conditions, this will be
2448      // handled by the LDIF reader, but this can happen if you call
2449      // decodeEntry with a set of data that includes a version.  At any rate,
2450      // read the next line, which must specify the DN.
2451      line = iterator.next();
2452      linesRead++;
2453      handleTrailingSpaces(line, null, firstLineNumber,
2454           unparsedRecord.getTrailingSpaceBehavior());
2455    }
2456
2457    colonPos = line.indexOf(":");
2458    if ((colonPos < 0) ||
2459         (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2460    {
2461      throw new LDIFException(
2462           ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2463           firstLineNumber, true, ldifLines, null);
2464    }
2465
2466    final String dn;
2467    final int length = line.length();
2468    if (length == (colonPos+1))
2469    {
2470      // The colon was the last character on the line.  This is acceptable and
2471      // indicates that the entry has the null DN.
2472      dn = "";
2473    }
2474    else if (line.charAt(colonPos+1) == ':')
2475    {
2476      // Skip over any spaces leading up to the value, and then the rest of the
2477      // string is the base64-encoded DN.
2478      int pos = colonPos+2;
2479      while ((pos < length) && (line.charAt(pos) == ' '))
2480      {
2481        pos++;
2482      }
2483
2484      try
2485      {
2486        final byte[] dnBytes = Base64.decode(line.substring(pos));
2487        dn = StaticUtils.toUTF8String(dnBytes);
2488      }
2489      catch (final ParseException pe)
2490      {
2491        Debug.debugException(pe);
2492        throw new LDIFException(
2493                       ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2494                                                               pe.getMessage()),
2495                       firstLineNumber, true, ldifLines, pe);
2496      }
2497      catch (final Exception e)
2498      {
2499        Debug.debugException(e);
2500        throw new LDIFException(
2501                       ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2502                                                               e),
2503                       firstLineNumber, true, ldifLines, e);
2504      }
2505    }
2506    else
2507    {
2508      // Skip over any spaces leading up to the value, and then the rest of the
2509      // string is the DN.
2510      int pos = colonPos+1;
2511      while ((pos < length) && (line.charAt(pos) == ' '))
2512      {
2513        pos++;
2514      }
2515
2516      dn = line.substring(pos);
2517    }
2518
2519
2520    // An LDIF change record may contain zero or more controls, with the end of
2521    // the controls signified by the changetype.  The changetype element must be
2522    // present, unless defaultAdd is true in which case the first thing that is
2523    // neither control or changetype will trigger the start of add attribute
2524    // parsing.
2525    if (! iterator.hasNext())
2526    {
2527      throw new LDIFException(ERR_READ_CR_TOO_SHORT.get(firstLineNumber),
2528                              firstLineNumber, true, ldifLines, null);
2529    }
2530
2531    String changeType;
2532    ArrayList<Control> controls = null;
2533    while (true)
2534    {
2535      line = iterator.next();
2536      handleTrailingSpaces(line, dn, firstLineNumber,
2537           unparsedRecord.getTrailingSpaceBehavior());
2538      colonPos = line.indexOf(":");
2539      if (colonPos < 0)
2540      {
2541        throw new LDIFException(
2542             ERR_READ_CR_SECOND_LINE_MISSING_COLON.get(firstLineNumber),
2543             firstLineNumber, true, ldifLines, null);
2544      }
2545
2546      final String token = StaticUtils.toLowerCase(line.substring(0, colonPos));
2547      if (token.equals("control"))
2548      {
2549        if (controls == null)
2550        {
2551          controls = new ArrayList<>(5);
2552        }
2553
2554        controls.add(decodeControl(line, colonPos, firstLineNumber, ldifLines,
2555             relativeBasePath));
2556      }
2557      else if (token.equals("changetype"))
2558      {
2559        changeType =
2560             decodeChangeType(line, colonPos, firstLineNumber, ldifLines);
2561        break;
2562      }
2563      else if (defaultAdd)
2564      {
2565        // The line we read wasn't a control or changetype declaration, so we'll
2566        // assume it's an attribute in an add record.  However, we're not ready
2567        // for that yet, and since we can't rewind an iterator we'll create a
2568        // new one that hasn't yet gotten to this line.
2569        changeType = "add";
2570        iterator = ldifLines.iterator();
2571        for (int i=0; i < linesRead; i++)
2572        {
2573          iterator.next();
2574        }
2575        break;
2576      }
2577      else
2578      {
2579        throw new LDIFException(
2580             ERR_READ_CR_CT_LINE_DOESNT_START_WITH_CONTROL_OR_CT.get(
2581                  firstLineNumber),
2582             firstLineNumber, true, ldifLines, null);
2583      }
2584
2585      linesRead++;
2586    }
2587
2588
2589    // Make sure that the change type is acceptable and then decode the rest of
2590    // the change record accordingly.
2591    final String lowerChangeType = StaticUtils.toLowerCase(changeType);
2592    if (lowerChangeType.equals("add"))
2593    {
2594      // There must be at least one more line.  If not, then that's an error.
2595      // Otherwise, parse the rest of the data as attribute-value pairs.
2596      if (iterator.hasNext())
2597      {
2598        final Collection<Attribute> attrs =
2599             parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2600                  unparsedRecord.getTrailingSpaceBehavior(),
2601                  unparsedRecord.getSchema(), ldifLines, iterator,
2602                  relativeBasePath, firstLineNumber);
2603        final Attribute[] attributes = new Attribute[attrs.size()];
2604        final Iterator<Attribute> attrIterator = attrs.iterator();
2605        for (int i=0; i < attributes.length; i++)
2606        {
2607          attributes[i] = attrIterator.next();
2608        }
2609
2610        return new LDIFAddChangeRecord(dn, attributes, controls);
2611      }
2612      else
2613      {
2614        throw new LDIFException(ERR_READ_CR_NO_ATTRIBUTES.get(firstLineNumber),
2615                                firstLineNumber, true, ldifLines, null);
2616      }
2617    }
2618    else if (lowerChangeType.equals("delete"))
2619    {
2620      // There shouldn't be any more data.  If there is, then that's an error.
2621      // Otherwise, we can just return the delete change record with what we
2622      // already know.
2623      if (iterator.hasNext())
2624      {
2625        throw new LDIFException(
2626                       ERR_READ_CR_EXTRA_DELETE_DATA.get(firstLineNumber),
2627                       firstLineNumber, true, ldifLines, null);
2628      }
2629      else
2630      {
2631        return new LDIFDeleteChangeRecord(dn, controls);
2632      }
2633    }
2634    else if (lowerChangeType.equals("modify"))
2635    {
2636      // There must be at least one more line.  If not, then that's an error.
2637      // Otherwise, parse the rest of the data as a set of modifications.
2638      if (iterator.hasNext())
2639      {
2640        final Modification[] mods = parseModifications(dn,
2641             unparsedRecord.getTrailingSpaceBehavior(), ldifLines, iterator,
2642             firstLineNumber, schema);
2643        return new LDIFModifyChangeRecord(dn, mods, controls);
2644      }
2645      else
2646      {
2647        throw new LDIFException(ERR_READ_CR_NO_MODS.get(firstLineNumber),
2648                                firstLineNumber, true, ldifLines, null);
2649      }
2650    }
2651    else if (lowerChangeType.equals("moddn") ||
2652             lowerChangeType.equals("modrdn"))
2653    {
2654      // There must be at least one more line.  If not, then that's an error.
2655      // Otherwise, parse the rest of the data as a set of modifications.
2656      if (iterator.hasNext())
2657      {
2658        return parseModifyDNChangeRecord(ldifLines, iterator, dn, controls,
2659             unparsedRecord.getTrailingSpaceBehavior(), firstLineNumber);
2660      }
2661      else
2662      {
2663        throw new LDIFException(ERR_READ_CR_NO_NEWRDN.get(firstLineNumber),
2664                                firstLineNumber, true, ldifLines, null);
2665      }
2666    }
2667    else
2668    {
2669      throw new LDIFException(ERR_READ_CR_INVALID_CT.get(changeType,
2670                                                         firstLineNumber),
2671                              firstLineNumber, true, ldifLines, null);
2672    }
2673  }
2674
2675
2676
2677  /**
2678   * Decodes information about a control from the provided line.
2679   *
2680   * @param  line              The line to process.
2681   * @param  colonPos          The position of the colon that separates the
2682   *                           control token string from tbe encoded control.
2683   * @param  firstLineNumber   The line number for the start of the record.
2684   * @param  ldifLines         The lines that comprise the LDIF representation
2685   *                           of the full record being parsed.
2686   * @param  relativeBasePath  The base path that will be prepended to relative
2687   *                           paths in order to obtain an absolute path.
2688   *
2689   * @return  The decoded control.
2690   *
2691   * @throws  LDIFException  If a problem is encountered while trying to decode
2692   *                         the changetype.
2693   */
2694  private static Control decodeControl(final StringBuilder line,
2695                                       final int colonPos,
2696                                       final long firstLineNumber,
2697                                       final ArrayList<StringBuilder> ldifLines,
2698                                       final String relativeBasePath)
2699          throws LDIFException
2700  {
2701    final String controlString;
2702    int length = line.length();
2703    if (length == (colonPos+1))
2704    {
2705      // The colon was the last character on the line.  This is not
2706      // acceptable.
2707      throw new LDIFException(
2708           ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2709           firstLineNumber, true, ldifLines, null);
2710    }
2711    else if (line.charAt(colonPos+1) == ':')
2712    {
2713      // Skip over any spaces leading up to the value, and then the rest of
2714      // the string is the base64-encoded control representation.  This is
2715      // unusual and unnecessary, but is nevertheless acceptable.
2716      int pos = colonPos+2;
2717      while ((pos < length) && (line.charAt(pos) == ' '))
2718      {
2719        pos++;
2720      }
2721
2722      try
2723      {
2724        final byte[] controlBytes = Base64.decode(line.substring(pos));
2725        controlString =  StaticUtils.toUTF8String(controlBytes);
2726      }
2727      catch (final ParseException pe)
2728      {
2729        Debug.debugException(pe);
2730        throw new LDIFException(
2731                       ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(
2732                            firstLineNumber, pe.getMessage()),
2733                       firstLineNumber, true, ldifLines, pe);
2734      }
2735      catch (final Exception e)
2736      {
2737        Debug.debugException(e);
2738        throw new LDIFException(
2739             ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(firstLineNumber, e),
2740             firstLineNumber, true, ldifLines, e);
2741      }
2742    }
2743    else
2744    {
2745      // Skip over any spaces leading up to the value, and then the rest of
2746      // the string is the encoded control.
2747      int pos = colonPos+1;
2748      while ((pos < length) && (line.charAt(pos) == ' '))
2749      {
2750        pos++;
2751      }
2752
2753      controlString = line.substring(pos);
2754    }
2755
2756    // If the resulting control definition is empty, then that's invalid.
2757    if (controlString.isEmpty())
2758    {
2759      throw new LDIFException(
2760           ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2761           firstLineNumber, true, ldifLines, null);
2762    }
2763
2764
2765    // The first element of the control must be the OID, and it must be followed
2766    // by a space (to separate it from the criticality), a colon (to separate it
2767    // from the value and indicate a default criticality of false), or the end
2768    // of the line (to indicate a default criticality of false and no value).
2769    String oid = null;
2770    boolean hasCriticality = false;
2771    boolean hasValue = false;
2772    int pos = 0;
2773    length = controlString.length();
2774    while (pos < length)
2775    {
2776      final char c = controlString.charAt(pos);
2777      if (c == ':')
2778      {
2779        // This indicates that there is no criticality and that the value
2780        // immediately follows the OID.
2781        oid = controlString.substring(0, pos++);
2782        hasValue = true;
2783        break;
2784      }
2785      else if (c == ' ')
2786      {
2787        // This indicates that there is a criticality.  We don't know anything
2788        // about the presence of a value yet.
2789        oid = controlString.substring(0, pos++);
2790        hasCriticality = true;
2791        break;
2792      }
2793      else
2794      {
2795        pos++;
2796      }
2797    }
2798
2799    if (oid == null)
2800    {
2801      // This indicates that the string representation of the control is only
2802      // the OID.
2803      return new Control(controlString, false);
2804    }
2805
2806
2807    // See if we need to read the criticality.  If so, then do so now.
2808    // Otherwise, assume a default criticality of false.
2809    final boolean isCritical;
2810    if (hasCriticality)
2811    {
2812      // Skip over any spaces before the criticality.
2813      while (controlString.charAt(pos) == ' ')
2814      {
2815        pos++;
2816      }
2817
2818      // Read until we find a colon or the end of the string.
2819      final int criticalityStartPos = pos;
2820      while (pos < length)
2821      {
2822        final char c = controlString.charAt(pos);
2823        if (c == ':')
2824        {
2825          hasValue = true;
2826          break;
2827        }
2828        else
2829        {
2830          pos++;
2831        }
2832      }
2833
2834      final String criticalityString =
2835           StaticUtils.toLowerCase(controlString.substring(criticalityStartPos,
2836                pos));
2837      if (criticalityString.equals("true"))
2838      {
2839        isCritical = true;
2840      }
2841      else if (criticalityString.equals("false"))
2842      {
2843        isCritical = false;
2844      }
2845      else
2846      {
2847        throw new LDIFException(
2848             ERR_READ_CONTROL_LINE_INVALID_CRITICALITY.get(criticalityString,
2849                  firstLineNumber),
2850             firstLineNumber, true, ldifLines, null);
2851      }
2852
2853      if (hasValue)
2854      {
2855        pos++;
2856      }
2857    }
2858    else
2859    {
2860      isCritical = false;
2861    }
2862
2863    // See if we need to read the value.  If so, then do so now.  It may be
2864    // a string, or it may be base64-encoded.  It could conceivably even be read
2865    // from a URL.
2866    final ASN1OctetString value;
2867    if (hasValue)
2868    {
2869      // The character immediately after the colon that precedes the value may
2870      // be one of the following:
2871      // - A second colon (optionally followed by a single space) to indicate
2872      //   that the value is base64-encoded.
2873      // - A less-than symbol to indicate that the value should be read from a
2874      //   location specified by a URL.
2875      // - A single space that precedes the non-base64-encoded value.
2876      // - The first character of the non-base64-encoded value.
2877      switch (controlString.charAt(pos))
2878      {
2879        case ':':
2880          try
2881          {
2882            if (controlString.length() == (pos+1))
2883            {
2884              value = new ASN1OctetString();
2885            }
2886            else if (controlString.charAt(pos+1) == ' ')
2887            {
2888              value = new ASN1OctetString(
2889                   Base64.decode(controlString.substring(pos+2)));
2890            }
2891            else
2892            {
2893              value = new ASN1OctetString(
2894                   Base64.decode(controlString.substring(pos+1)));
2895            }
2896          }
2897          catch (final Exception e)
2898          {
2899            Debug.debugException(e);
2900            throw new LDIFException(
2901                 ERR_READ_CONTROL_LINE_CANNOT_BASE64_DECODE_VALUE.get(
2902                      firstLineNumber, StaticUtils.getExceptionMessage(e)),
2903                 firstLineNumber, true, ldifLines, e);
2904          }
2905          break;
2906        case '<':
2907          try
2908          {
2909            final String urlString;
2910            if (controlString.charAt(pos+1) == ' ')
2911            {
2912              urlString = controlString.substring(pos+2);
2913            }
2914            else
2915            {
2916              urlString = controlString.substring(pos+1);
2917            }
2918            value = new ASN1OctetString(retrieveURLBytes(urlString,
2919                 relativeBasePath, firstLineNumber));
2920          }
2921          catch (final Exception e)
2922          {
2923            Debug.debugException(e);
2924            throw new LDIFException(
2925                 ERR_READ_CONTROL_LINE_CANNOT_RETRIEVE_VALUE_FROM_URL.get(
2926                      firstLineNumber, StaticUtils.getExceptionMessage(e)),
2927                 firstLineNumber, true, ldifLines, e);
2928          }
2929          break;
2930        case ' ':
2931          value = new ASN1OctetString(controlString.substring(pos+1));
2932          break;
2933        default:
2934          value = new ASN1OctetString(controlString.substring(pos));
2935          break;
2936      }
2937    }
2938    else
2939    {
2940      value = null;
2941    }
2942
2943    return new Control(oid, isCritical, value);
2944  }
2945
2946
2947
2948  /**
2949   * Decodes the changetype element from the provided line.
2950   *
2951   * @param  line             The line to process.
2952   * @param  colonPos         The position of the colon that separates the
2953   *                          changetype string from its value.
2954   * @param  firstLineNumber  The line number for the start of the record.
2955   * @param  ldifLines        The lines that comprise the LDIF representation of
2956   *                          the full record being parsed.
2957   *
2958   * @return  The decoded changetype string.
2959   *
2960   * @throws  LDIFException  If a problem is encountered while trying to decode
2961   *                         the changetype.
2962   */
2963  private static String decodeChangeType(final StringBuilder line,
2964                             final int colonPos, final long firstLineNumber,
2965                             final ArrayList<StringBuilder> ldifLines)
2966          throws LDIFException
2967  {
2968    final int length = line.length();
2969    if (length == (colonPos+1))
2970    {
2971      // The colon was the last character on the line.  This is not
2972      // acceptable.
2973      throw new LDIFException(
2974           ERR_READ_CT_LINE_NO_CT_VALUE.get(firstLineNumber), firstLineNumber,
2975           true, ldifLines, null);
2976    }
2977    else if (line.charAt(colonPos+1) == ':')
2978    {
2979      // Skip over any spaces leading up to the value, and then the rest of
2980      // the string is the base64-encoded changetype.  This is unusual and
2981      // unnecessary, but is nevertheless acceptable.
2982      int pos = colonPos+2;
2983      while ((pos < length) && (line.charAt(pos) == ' '))
2984      {
2985        pos++;
2986      }
2987
2988      try
2989      {
2990        final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
2991        return StaticUtils.toUTF8String(changeTypeBytes);
2992      }
2993      catch (final ParseException pe)
2994      {
2995        Debug.debugException(pe);
2996        throw new LDIFException(
2997                       ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber,
2998                                                            pe.getMessage()),
2999                       firstLineNumber, true, ldifLines, pe);
3000      }
3001      catch (final Exception e)
3002      {
3003        Debug.debugException(e);
3004        throw new LDIFException(
3005             ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, e),
3006             firstLineNumber, true, ldifLines, e);
3007      }
3008    }
3009    else
3010    {
3011      // Skip over any spaces leading up to the value, and then the rest of
3012      // the string is the changetype.
3013      int pos = colonPos+1;
3014      while ((pos < length) && (line.charAt(pos) == ' '))
3015      {
3016        pos++;
3017      }
3018
3019      return line.substring(pos);
3020    }
3021  }
3022
3023
3024
3025  /**
3026   * Parses the data available through the provided iterator as a collection of
3027   * attributes suitable for use in an entry or an add change record.
3028   *
3029   * @param  dn                      The DN of the record being read.
3030   * @param  duplicateValueBehavior  The behavior that should be exhibited if
3031   *                                 the LDIF reader encounters an entry with
3032   *                                 duplicate values.
3033   * @param  trailingSpaceBehavior   The behavior that should be exhibited when
3034   *                                 encountering attribute values which are not
3035   *                                 base64-encoded but contain trailing spaces.
3036   * @param  schema                  The schema to use when parsing the
3037   *                                 attributes, or {@code null} if none is
3038   *                                 needed.
3039   * @param  ldifLines               The lines that comprise the LDIF
3040   *                                 representation of the full record being
3041   *                                 parsed.
3042   * @param  iterator                The iterator to use to access the attribute
3043   *                                 lines.
3044   * @param  relativeBasePath        The base path that will be prepended to
3045   *                                 relative paths in order to obtain an
3046   *                                 absolute path.
3047   * @param  firstLineNumber         The line number for the start of the
3048   *                                 record.
3049   *
3050   * @return  The collection of attributes that were read.
3051   *
3052   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3053   *                         set of attributes.
3054   */
3055  private static ArrayList<Attribute> parseAttributes(final String dn,
3056       final DuplicateValueBehavior duplicateValueBehavior,
3057       final TrailingSpaceBehavior trailingSpaceBehavior, final Schema schema,
3058       final ArrayList<StringBuilder> ldifLines,
3059       final Iterator<StringBuilder> iterator, final String relativeBasePath,
3060       final long firstLineNumber)
3061          throws LDIFException
3062  {
3063    final LinkedHashMap<String,Object> attributes =
3064         new LinkedHashMap<>(StaticUtils.computeMapCapacity(ldifLines.size()));
3065    while (iterator.hasNext())
3066    {
3067      final StringBuilder line = iterator.next();
3068      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3069      final int colonPos = line.indexOf(":");
3070      if (colonPos <= 0)
3071      {
3072        throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3073                                firstLineNumber, true, ldifLines, null);
3074      }
3075
3076      final String attributeName = line.substring(0, colonPos);
3077      final String lowerName     = StaticUtils.toLowerCase(attributeName);
3078
3079      final MatchingRule matchingRule;
3080      if (schema == null)
3081      {
3082        matchingRule = CaseIgnoreStringMatchingRule.getInstance();
3083      }
3084      else
3085      {
3086        matchingRule =
3087             MatchingRule.selectEqualityMatchingRule(attributeName, schema);
3088      }
3089
3090      Attribute attr;
3091      final LDIFAttribute ldifAttr;
3092      final Object attrObject = attributes.get(lowerName);
3093      if (attrObject == null)
3094      {
3095        attr     = null;
3096        ldifAttr = null;
3097      }
3098      else
3099      {
3100        if (attrObject instanceof Attribute)
3101        {
3102          attr     = (Attribute) attrObject;
3103          ldifAttr = new LDIFAttribute(attr.getName(), matchingRule,
3104                                       attr.getRawValues()[0]);
3105          attributes.put(lowerName, ldifAttr);
3106        }
3107        else
3108        {
3109          attr     = null;
3110          ldifAttr = (LDIFAttribute) attrObject;
3111        }
3112      }
3113
3114      final int length = line.length();
3115      if (length == (colonPos+1))
3116      {
3117        // This means that the attribute has a zero-length value, which is
3118        // acceptable.
3119        if (attrObject == null)
3120        {
3121          attr = new Attribute(attributeName, matchingRule, "");
3122          attributes.put(lowerName, attr);
3123        }
3124        else
3125        {
3126          try
3127          {
3128            if (! ldifAttr.addValue(new ASN1OctetString(),
3129                       duplicateValueBehavior))
3130            {
3131              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3132              {
3133                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3134                     firstLineNumber, attributeName), firstLineNumber, true,
3135                     ldifLines, null);
3136              }
3137            }
3138          }
3139          catch (final LDAPException le)
3140          {
3141            throw new LDIFException(
3142                 ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, firstLineNumber,
3143                      attributeName, StaticUtils.getExceptionMessage(le)),
3144                 firstLineNumber, true, ldifLines, le);
3145          }
3146        }
3147      }
3148      else if (line.charAt(colonPos+1) == ':')
3149      {
3150        // Skip over any spaces leading up to the value, and then the rest of
3151        // the string is the base64-encoded attribute value.
3152        int pos = colonPos+2;
3153        while ((pos < length) && (line.charAt(pos) == ' '))
3154        {
3155          pos++;
3156        }
3157
3158        try
3159        {
3160          final byte[] valueBytes = Base64.decode(line.substring(pos));
3161          if (attrObject == null)
3162          {
3163            attr = new Attribute(attributeName, matchingRule, valueBytes);
3164            attributes.put(lowerName, attr);
3165          }
3166          else
3167          {
3168            try
3169            {
3170              if (! ldifAttr.addValue(new ASN1OctetString(valueBytes),
3171                         duplicateValueBehavior))
3172              {
3173                if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3174                {
3175                  throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3176                       firstLineNumber, attributeName), firstLineNumber, true,
3177                       ldifLines, null);
3178                }
3179              }
3180            }
3181            catch (final LDAPException le)
3182            {
3183              throw new LDIFException(
3184                   ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, firstLineNumber,
3185                        attributeName, StaticUtils.getExceptionMessage(le)),
3186                   firstLineNumber, true, ldifLines, le);
3187            }
3188          }
3189        }
3190        catch (final ParseException pe)
3191        {
3192          Debug.debugException(pe);
3193          throw new LDIFException(
3194               ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(attributeName,
3195                    firstLineNumber, pe.getMessage()),
3196               firstLineNumber, true, ldifLines, pe);
3197        }
3198      }
3199      else if (line.charAt(colonPos+1) == '<')
3200      {
3201        // Skip over any spaces leading up to the value, and then the rest of
3202        // the string is a URL that indicates where to get the real content.
3203        // At the present time, we'll only support the file URLs.
3204        int pos = colonPos+2;
3205        while ((pos < length) && (line.charAt(pos) == ' '))
3206        {
3207          pos++;
3208        }
3209
3210        final byte[] urlBytes;
3211        final String urlString = line.substring(pos);
3212        try
3213        {
3214          urlBytes =
3215               retrieveURLBytes(urlString, relativeBasePath, firstLineNumber);
3216        }
3217        catch (final Exception e)
3218        {
3219          Debug.debugException(e);
3220          throw new LDIFException(
3221               ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3222                    firstLineNumber, e),
3223               firstLineNumber, true, ldifLines, e);
3224        }
3225
3226        if (attrObject == null)
3227        {
3228          attr = new Attribute(attributeName, matchingRule, urlBytes);
3229          attributes.put(lowerName, attr);
3230        }
3231        else
3232        {
3233          try
3234          {
3235            if (! ldifAttr.addValue(new ASN1OctetString(urlBytes),
3236                 duplicateValueBehavior))
3237            {
3238              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3239              {
3240                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3241                     firstLineNumber, attributeName), firstLineNumber, true,
3242                     ldifLines, null);
3243              }
3244            }
3245          }
3246          catch (final LDIFException le)
3247          {
3248            Debug.debugException(le);
3249            throw le;
3250          }
3251          catch (final Exception e)
3252          {
3253            Debug.debugException(e);
3254            throw new LDIFException(
3255                 ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3256                      firstLineNumber, e),
3257                 firstLineNumber, true, ldifLines, e);
3258          }
3259        }
3260      }
3261      else
3262      {
3263        // Skip over any spaces leading up to the value, and then the rest of
3264        // the string is the value.
3265        int pos = colonPos+1;
3266        while ((pos < length) && (line.charAt(pos) == ' '))
3267        {
3268          pos++;
3269        }
3270
3271        final String valueString = line.substring(pos);
3272        if (attrObject == null)
3273        {
3274          attr = new Attribute(attributeName, matchingRule, valueString);
3275          attributes.put(lowerName, attr);
3276        }
3277        else
3278        {
3279          try
3280          {
3281            if (! ldifAttr.addValue(new ASN1OctetString(valueString),
3282                       duplicateValueBehavior))
3283            {
3284              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3285              {
3286                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3287                     firstLineNumber, attributeName), firstLineNumber, true,
3288                     ldifLines, null);
3289              }
3290            }
3291          }
3292          catch (final LDAPException le)
3293          {
3294            throw new LDIFException(
3295                 ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, firstLineNumber,
3296                      attributeName, StaticUtils.getExceptionMessage(le)),
3297                 firstLineNumber, true, ldifLines, le);
3298          }
3299        }
3300      }
3301    }
3302
3303    final ArrayList<Attribute> attrList = new ArrayList<>(attributes.size());
3304    for (final Object o : attributes.values())
3305    {
3306      if (o instanceof Attribute)
3307      {
3308        attrList.add((Attribute) o);
3309      }
3310      else
3311      {
3312        attrList.add(((LDIFAttribute) o).toAttribute());
3313      }
3314    }
3315
3316    return attrList;
3317  }
3318
3319
3320
3321  /**
3322   * Retrieves the bytes that make up the file referenced by the given URL.
3323   *
3324   * @param  urlString         The string representation of the URL to retrieve.
3325   * @param  relativeBasePath  The base path that will be prepended to relative
3326   *                           paths in order to obtain an absolute path.
3327   * @param  firstLineNumber   The line number for the start of the record.
3328   *
3329   * @return  The bytes contained in the specified file, or an empty array if
3330   *          the specified file is empty.
3331   *
3332   * @throws  LDIFException  If the provided URL is malformed or references a
3333   *                         nonexistent file.
3334   *
3335   * @throws  IOException  If a problem is encountered while attempting to read
3336   *                       from the target file.
3337   */
3338  private static byte[] retrieveURLBytes(final String urlString,
3339                                         final String relativeBasePath,
3340                                         final long firstLineNumber)
3341          throws LDIFException, IOException
3342  {
3343    int pos;
3344    final String path;
3345    final String lowerURLString = StaticUtils.toLowerCase(urlString);
3346    if (lowerURLString.startsWith("file:/"))
3347    {
3348      pos = 6;
3349      while ((pos < urlString.length()) && (urlString.charAt(pos) == '/'))
3350      {
3351        pos++;
3352      }
3353
3354      path = urlString.substring(pos-1);
3355    }
3356    else if (lowerURLString.startsWith("file:"))
3357    {
3358      // A file: URL that doesn't include a slash will be interpreted as a
3359      // relative path.
3360      path = relativeBasePath + urlString.substring(5);
3361    }
3362    else
3363    {
3364      throw new LDIFException(ERR_READ_URL_INVALID_SCHEME.get(urlString),
3365           firstLineNumber, true);
3366    }
3367
3368    final File f = new File(path);
3369    if (! f.exists())
3370    {
3371      throw new LDIFException(
3372           ERR_READ_URL_NO_SUCH_FILE.get(urlString, f.getAbsolutePath()),
3373           firstLineNumber, true);
3374    }
3375
3376    // In order to conserve memory, we'll only allow values to be read from
3377    // files no larger than 10 megabytes.
3378    final long fileSize = f.length();
3379    if (fileSize > (10 * 1024 * 1024))
3380    {
3381      throw new LDIFException(
3382           ERR_READ_URL_FILE_TOO_LARGE.get(urlString, f.getAbsolutePath(),
3383                (10*1024*1024)),
3384           firstLineNumber, true);
3385    }
3386
3387    int fileBytesRemaining = (int) fileSize;
3388    final byte[] fileData = new byte[(int) fileSize];
3389    final FileInputStream fis = new FileInputStream(f);
3390    try
3391    {
3392      int fileBytesRead = 0;
3393      while (fileBytesRead < fileSize)
3394      {
3395        final int bytesRead =
3396             fis.read(fileData, fileBytesRead, fileBytesRemaining);
3397        if (bytesRead < 0)
3398        {
3399          // We hit the end of the file before we expected to.  This shouldn't
3400          // happen unless the file size changed since we first looked at it,
3401          // which we won't allow.
3402          throw new LDIFException(
3403               ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString,
3404                    f.getAbsolutePath()),
3405               firstLineNumber, true);
3406        }
3407
3408        fileBytesRead      += bytesRead;
3409        fileBytesRemaining -= bytesRead;
3410      }
3411
3412      if (fis.read() != -1)
3413      {
3414        // There is still more data to read.  This shouldn't happen unless the
3415        // file size changed since we first looked at it, which we won't allow.
3416        throw new LDIFException(
3417             ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, f.getAbsolutePath()),
3418             firstLineNumber, true);
3419      }
3420    }
3421    finally
3422    {
3423      fis.close();
3424    }
3425
3426    return fileData;
3427  }
3428
3429
3430
3431  /**
3432   * Parses the data available through the provided iterator into an array of
3433   * modifications suitable for use in a modify change record.
3434   *
3435   * @param  dn                     The DN of the entry being parsed.
3436   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3437   *                                encountering attribute values which are not
3438   *                                base64-encoded but contain trailing spaces.
3439   * @param  ldifLines              The lines that comprise the LDIF
3440   *                                representation of the full record being
3441   *                                parsed.
3442   * @param  iterator               The iterator to use to access the
3443   *                                modification data.
3444   * @param  firstLineNumber        The line number for the start of the record.
3445   * @param  schema                 The schema to use in processing.
3446   *
3447   * @return  An array containing the modifications that were read.
3448   *
3449   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3450   *                         set of modifications.
3451   */
3452  private static Modification[] parseModifications(final String dn,
3453                      final TrailingSpaceBehavior trailingSpaceBehavior,
3454                      final ArrayList<StringBuilder> ldifLines,
3455                      final Iterator<StringBuilder> iterator,
3456                      final long firstLineNumber, final Schema schema)
3457          throws LDIFException
3458  {
3459    final ArrayList<Modification> modList = new ArrayList<>(ldifLines.size());
3460
3461    while (iterator.hasNext())
3462    {
3463      // The first line must start with "add:", "delete:", "replace:", or
3464      // "increment:" followed by an attribute name.
3465      StringBuilder line = iterator.next();
3466      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3467      int colonPos = line.indexOf(":");
3468      if (colonPos < 0)
3469      {
3470        throw new LDIFException(ERR_READ_MOD_CR_NO_MODTYPE.get(firstLineNumber),
3471                                firstLineNumber, true, ldifLines, null);
3472      }
3473
3474      final ModificationType modType;
3475      final String modTypeStr =
3476           StaticUtils.toLowerCase(line.substring(0, colonPos));
3477      if (modTypeStr.equals("add"))
3478      {
3479        modType = ModificationType.ADD;
3480      }
3481      else if (modTypeStr.equals("delete"))
3482      {
3483        modType = ModificationType.DELETE;
3484      }
3485      else if (modTypeStr.equals("replace"))
3486      {
3487        modType = ModificationType.REPLACE;
3488      }
3489      else if (modTypeStr.equals("increment"))
3490      {
3491        modType = ModificationType.INCREMENT;
3492      }
3493      else
3494      {
3495        throw new LDIFException(ERR_READ_MOD_CR_INVALID_MODTYPE.get(modTypeStr,
3496                                     firstLineNumber),
3497                                firstLineNumber, true, ldifLines, null);
3498      }
3499
3500      String attributeName;
3501      int length = line.length();
3502      if (length == (colonPos+1))
3503      {
3504        // The colon was the last character on the line.  This is not
3505        // acceptable.
3506        throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3507                                     firstLineNumber),
3508                                firstLineNumber, true, ldifLines, null);
3509      }
3510      else if (line.charAt(colonPos+1) == ':')
3511      {
3512        // Skip over any spaces leading up to the value, and then the rest of
3513        // the string is the base64-encoded attribute name.
3514        int pos = colonPos+2;
3515        while ((pos < length) && (line.charAt(pos) == ' '))
3516        {
3517          pos++;
3518        }
3519
3520        try
3521        {
3522          final byte[] dnBytes = Base64.decode(line.substring(pos));
3523          attributeName = StaticUtils.toUTF8String(dnBytes);
3524        }
3525        catch (final ParseException pe)
3526        {
3527          Debug.debugException(pe);
3528          throw new LDIFException(
3529               ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3530                    firstLineNumber, pe.getMessage()),
3531               firstLineNumber, true, ldifLines, pe);
3532        }
3533        catch (final Exception e)
3534        {
3535          Debug.debugException(e);
3536          throw new LDIFException(
3537               ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3538                    firstLineNumber, e),
3539               firstLineNumber, true, ldifLines, e);
3540        }
3541      }
3542      else
3543      {
3544        // Skip over any spaces leading up to the value, and then the rest of
3545        // the string is the attribute name.
3546        int pos = colonPos+1;
3547        while ((pos < length) && (line.charAt(pos) == ' '))
3548        {
3549          pos++;
3550        }
3551
3552        attributeName = line.substring(pos);
3553      }
3554
3555      if (attributeName.isEmpty())
3556      {
3557        throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3558                                     firstLineNumber),
3559                                firstLineNumber, true, ldifLines, null);
3560      }
3561
3562
3563      // The next zero or more lines may be the set of attribute values.  Keep
3564      // reading until we reach the end of the iterator or until we find a line
3565      // with just a "-".
3566      final ArrayList<ASN1OctetString> valueList =
3567           new ArrayList<>(ldifLines.size());
3568      while (iterator.hasNext())
3569      {
3570        line = iterator.next();
3571        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3572        if (line.toString().equals("-"))
3573        {
3574          break;
3575        }
3576
3577        colonPos = line.indexOf(":");
3578        if (colonPos < 0)
3579        {
3580          throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3581                                  firstLineNumber, true, ldifLines, null);
3582        }
3583        else if (! line.substring(0, colonPos).equalsIgnoreCase(attributeName))
3584        {
3585          // There are a couple of cases in which this might be acceptable:
3586          // - If the two names are logically equivalent, but have an alternate
3587          //   name (or OID) for the target attribute type, or if there are
3588          //   attribute options and the options are just in a different order.
3589          // - If this is the first value for the target attribute and the
3590          //   alternate name includes a "binary" option that the original
3591          //   attribute name did not have.  In this case, all subsequent values
3592          //   will also be required to have the binary option.
3593          final String alternateName = line.substring(0, colonPos);
3594
3595
3596          // Check to see if the base names are equivalent.
3597          boolean baseNameEquivalent = false;
3598          final String expectedBaseName = Attribute.getBaseName(attributeName);
3599          final String alternateBaseName = Attribute.getBaseName(alternateName);
3600          if (alternateBaseName.equalsIgnoreCase(expectedBaseName))
3601          {
3602            baseNameEquivalent = true;
3603          }
3604          else
3605          {
3606            if (schema != null)
3607            {
3608              final AttributeTypeDefinition expectedAT =
3609                   schema.getAttributeType(expectedBaseName);
3610              final AttributeTypeDefinition alternateAT =
3611                   schema.getAttributeType(alternateBaseName);
3612              if ((expectedAT != null) && (alternateAT != null) &&
3613                  expectedAT.equals(alternateAT))
3614              {
3615                baseNameEquivalent = true;
3616              }
3617            }
3618          }
3619
3620
3621          // Check to see if the attribute options are equivalent.
3622          final Set<String> expectedOptions =
3623               Attribute.getOptions(attributeName);
3624          final Set<String> lowerExpectedOptions = new HashSet<>(
3625               StaticUtils.computeMapCapacity(expectedOptions.size()));
3626          for (final String s : expectedOptions)
3627          {
3628            lowerExpectedOptions.add(StaticUtils.toLowerCase(s));
3629          }
3630
3631          final Set<String> alternateOptions =
3632               Attribute.getOptions(alternateName);
3633          final Set<String> lowerAlternateOptions = new HashSet<>(
3634               StaticUtils.computeMapCapacity(alternateOptions.size()));
3635          for (final String s : alternateOptions)
3636          {
3637            lowerAlternateOptions.add(StaticUtils.toLowerCase(s));
3638          }
3639
3640          final boolean optionsEquivalent =
3641               lowerAlternateOptions.equals(lowerExpectedOptions);
3642
3643
3644          if (baseNameEquivalent && optionsEquivalent)
3645          {
3646            // This is fine.  The two attribute descriptions are logically
3647            // equivalent.  We'll continue using the attribute description that
3648            // was provided first.
3649          }
3650          else if (valueList.isEmpty() && baseNameEquivalent &&
3651                   lowerAlternateOptions.remove("binary") &&
3652                   lowerAlternateOptions.equals(lowerExpectedOptions))
3653          {
3654            // This means that the provided value is the first value for the
3655            // attribute, and that the only significant difference is that the
3656            // provided attribute description included an unexpected "binary"
3657            // option.  We'll accept this, but will require any additional
3658            // values for this modification to also include the binary option,
3659            // and we'll use the binary option in the attribute that is
3660            // eventually created.
3661            attributeName = alternateName;
3662          }
3663          else
3664          {
3665            // This means that either the base names are different or the sets
3666            // of options are incompatible.  This is not acceptable.
3667            throw new LDIFException(ERR_READ_MOD_CR_ATTR_MISMATCH.get(
3668                                         firstLineNumber,
3669                                         line.substring(0, colonPos),
3670                                         attributeName),
3671                                    firstLineNumber, true, ldifLines, null);
3672          }
3673        }
3674
3675        length = line.length();
3676        final ASN1OctetString value;
3677        if (length == (colonPos+1))
3678        {
3679          // The colon was the last character on the line.  This is fine.
3680          value = new ASN1OctetString();
3681        }
3682        else if (line.charAt(colonPos+1) == ':')
3683        {
3684          // Skip over any spaces leading up to the value, and then the rest of
3685          // the string is the base64-encoded value.  This is unusual and
3686          // unnecessary, but is nevertheless acceptable.
3687          int pos = colonPos+2;
3688          while ((pos < length) && (line.charAt(pos) == ' '))
3689          {
3690            pos++;
3691          }
3692
3693          try
3694          {
3695            value = new ASN1OctetString(Base64.decode(line.substring(pos)));
3696          }
3697          catch (final ParseException pe)
3698          {
3699            Debug.debugException(pe);
3700            throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3701                 attributeName, firstLineNumber, pe.getMessage()),
3702                 firstLineNumber, true, ldifLines, pe);
3703          }
3704          catch (final Exception e)
3705          {
3706            Debug.debugException(e);
3707            throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3708                                         firstLineNumber, e),
3709                                    firstLineNumber, true, ldifLines, e);
3710          }
3711        }
3712        else
3713        {
3714          // Skip over any spaces leading up to the value, and then the rest of
3715          // the string is the value.
3716          int pos = colonPos+1;
3717          while ((pos < length) && (line.charAt(pos) == ' '))
3718          {
3719            pos++;
3720          }
3721
3722          value = new ASN1OctetString(line.substring(pos));
3723        }
3724
3725        valueList.add(value);
3726      }
3727
3728      final ASN1OctetString[] values = new ASN1OctetString[valueList.size()];
3729      valueList.toArray(values);
3730
3731      // If it's an add modification type, then there must be at least one
3732      // value.
3733      if ((modType.intValue() == ModificationType.ADD.intValue()) &&
3734          (values.length == 0))
3735      {
3736        throw new LDIFException(ERR_READ_MOD_CR_NO_ADD_VALUES.get(attributeName,
3737                                     firstLineNumber),
3738                                firstLineNumber, true, ldifLines, null);
3739      }
3740
3741      // If it's an increment modification type, then there must be exactly one
3742      // value.
3743      if ((modType.intValue() == ModificationType.INCREMENT.intValue()) &&
3744          (values.length != 1))
3745      {
3746        throw new LDIFException(ERR_READ_MOD_CR_INVALID_INCR_VALUE_COUNT.get(
3747                                     firstLineNumber, attributeName),
3748                                firstLineNumber, true, ldifLines, null);
3749      }
3750
3751      modList.add(new Modification(modType, attributeName, values));
3752    }
3753
3754    final Modification[] mods = new Modification[modList.size()];
3755    modList.toArray(mods);
3756    return mods;
3757  }
3758
3759
3760
3761  /**
3762   * Parses the data available through the provided iterator as the body of a
3763   * modify DN change record (i.e., the newrdn, deleteoldrdn, and optional
3764   * newsuperior lines).
3765   *
3766   * @param  ldifLines              The lines that comprise the LDIF
3767   *                                representation of the full record being
3768   *                                parsed.
3769   * @param  iterator               The iterator to use to access the modify DN
3770   *                                data.
3771   * @param  dn                     The current DN of the entry.
3772   * @param  controls               The set of controls to include in the change
3773   *                                record.
3774   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3775   *                                encountering attribute values which are not
3776   *                                base64-encoded but contain trailing spaces.
3777   * @param  firstLineNumber        The line number for the start of the record.
3778   *
3779   * @return  The decoded modify DN change record.
3780   *
3781   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3782   *                         modify DN change record.
3783   */
3784  private static LDIFModifyDNChangeRecord parseModifyDNChangeRecord(
3785       final ArrayList<StringBuilder> ldifLines,
3786       final Iterator<StringBuilder> iterator, final String dn,
3787       final List<Control> controls,
3788       final TrailingSpaceBehavior trailingSpaceBehavior,
3789       final long firstLineNumber)
3790       throws LDIFException
3791  {
3792    // The next line must be the new RDN, and it must start with "newrdn:".
3793    StringBuilder line = iterator.next();
3794    handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3795    int colonPos = line.indexOf(":");
3796    if ((colonPos < 0) ||
3797        (! line.substring(0, colonPos).equalsIgnoreCase("newrdn")))
3798    {
3799      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_COLON.get(
3800                                   firstLineNumber),
3801                              firstLineNumber, true, ldifLines, null);
3802    }
3803
3804    final String newRDN;
3805    int length = line.length();
3806    if (length == (colonPos+1))
3807    {
3808      // The colon was the last character on the line.  This is not acceptable.
3809      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3810                                   firstLineNumber),
3811                              firstLineNumber, true, ldifLines, null);
3812    }
3813    else if (line.charAt(colonPos+1) == ':')
3814    {
3815      // Skip over any spaces leading up to the value, and then the rest of the
3816      // string is the base64-encoded new RDN.
3817      int pos = colonPos+2;
3818      while ((pos < length) && (line.charAt(pos) == ' '))
3819      {
3820        pos++;
3821      }
3822
3823      try
3824      {
3825        final byte[] dnBytes = Base64.decode(line.substring(pos));
3826        newRDN = StaticUtils.toUTF8String(dnBytes);
3827      }
3828      catch (final ParseException pe)
3829      {
3830        Debug.debugException(pe);
3831        throw new LDIFException(
3832             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3833                                                               pe.getMessage()),
3834             firstLineNumber, true, ldifLines, pe);
3835      }
3836      catch (final Exception e)
3837      {
3838        Debug.debugException(e);
3839        throw new LDIFException(
3840             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3841                                                               e),
3842             firstLineNumber, true, ldifLines, e);
3843      }
3844    }
3845    else
3846    {
3847      // Skip over any spaces leading up to the value, and then the rest of the
3848      // string is the new RDN.
3849      int pos = colonPos+1;
3850      while ((pos < length) && (line.charAt(pos) == ' '))
3851      {
3852        pos++;
3853      }
3854
3855      newRDN = line.substring(pos);
3856    }
3857
3858    if (newRDN.isEmpty())
3859    {
3860      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3861                                   firstLineNumber),
3862                              firstLineNumber, true, ldifLines, null);
3863    }
3864
3865
3866    // The next line must be the deleteOldRDN flag, and it must start with
3867    // 'deleteoldrdn:'.
3868    if (! iterator.hasNext())
3869    {
3870      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3871                                   firstLineNumber),
3872                              firstLineNumber, true, ldifLines, null);
3873    }
3874
3875    line = iterator.next();
3876    handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3877    colonPos = line.indexOf(":");
3878    if ((colonPos < 0) ||
3879        (! line.substring(0, colonPos).equalsIgnoreCase("deleteoldrdn")))
3880    {
3881      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3882                                   firstLineNumber),
3883                              firstLineNumber, true, ldifLines, null);
3884    }
3885
3886    final String deleteOldRDNStr;
3887    length = line.length();
3888    if (length == (colonPos+1))
3889    {
3890      // The colon was the last character on the line.  This is not acceptable.
3891      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_VALUE.get(
3892                                   firstLineNumber),
3893                              firstLineNumber, true, ldifLines, null);
3894    }
3895    else if (line.charAt(colonPos+1) == ':')
3896    {
3897      // Skip over any spaces leading up to the value, and then the rest of the
3898      // string is the base64-encoded value.  This is unusual and
3899      // unnecessary, but is nevertheless acceptable.
3900      int pos = colonPos+2;
3901      while ((pos < length) && (line.charAt(pos) == ' '))
3902      {
3903        pos++;
3904      }
3905
3906      try
3907      {
3908        final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
3909        deleteOldRDNStr = StaticUtils.toUTF8String(changeTypeBytes);
3910      }
3911      catch (final ParseException pe)
3912      {
3913        Debug.debugException(pe);
3914        throw new LDIFException(
3915             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3916                  firstLineNumber, pe.getMessage()),
3917             firstLineNumber, true, ldifLines, pe);
3918      }
3919      catch (final Exception e)
3920      {
3921        Debug.debugException(e);
3922        throw new LDIFException(
3923             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3924                  firstLineNumber, e),
3925             firstLineNumber, true, ldifLines, e);
3926      }
3927    }
3928    else
3929    {
3930      // Skip over any spaces leading up to the value, and then the rest of the
3931      // string is the value.
3932      int pos = colonPos+1;
3933      while ((pos < length) && (line.charAt(pos) == ' '))
3934      {
3935        pos++;
3936      }
3937
3938      deleteOldRDNStr = line.substring(pos);
3939    }
3940
3941    final boolean deleteOldRDN;
3942    if (deleteOldRDNStr.equals("0"))
3943    {
3944      deleteOldRDN = false;
3945    }
3946    else if (deleteOldRDNStr.equals("1"))
3947    {
3948      deleteOldRDN = true;
3949    }
3950    else if (deleteOldRDNStr.equalsIgnoreCase("false") ||
3951             deleteOldRDNStr.equalsIgnoreCase("no"))
3952    {
3953      // This is technically illegal, but we'll allow it.
3954      deleteOldRDN = false;
3955    }
3956    else if (deleteOldRDNStr.equalsIgnoreCase("true") ||
3957             deleteOldRDNStr.equalsIgnoreCase("yes"))
3958    {
3959      // This is also technically illegal, but we'll allow it.
3960      deleteOldRDN = false;
3961    }
3962    else
3963    {
3964      throw new LDIFException(ERR_READ_MODDN_CR_INVALID_DELOLDRDN.get(
3965                                   deleteOldRDNStr, firstLineNumber),
3966                              firstLineNumber, true, ldifLines, null);
3967    }
3968
3969
3970    // If there is another line, then it must be the new superior DN and it must
3971    // start with "newsuperior:".  If this is absent, then it's fine.
3972    final String newSuperiorDN;
3973    if (iterator.hasNext())
3974    {
3975      line = iterator.next();
3976      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3977      colonPos = line.indexOf(":");
3978      if ((colonPos < 0) ||
3979          (! line.substring(0, colonPos).equalsIgnoreCase("newsuperior")))
3980      {
3981        throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWSUPERIOR_COLON.get(
3982                                     firstLineNumber),
3983                                firstLineNumber, true, ldifLines, null);
3984      }
3985
3986      length = line.length();
3987      if (length == (colonPos+1))
3988      {
3989        // The colon was the last character on the line.  This is fine.
3990        newSuperiorDN = "";
3991      }
3992      else if (line.charAt(colonPos+1) == ':')
3993      {
3994        // Skip over any spaces leading up to the value, and then the rest of
3995        // the string is the base64-encoded new superior DN.
3996        int pos = colonPos+2;
3997        while ((pos < length) && (line.charAt(pos) == ' '))
3998        {
3999          pos++;
4000        }
4001
4002        try
4003        {
4004          final byte[] dnBytes = Base64.decode(line.substring(pos));
4005          newSuperiorDN = StaticUtils.toUTF8String(dnBytes);
4006        }
4007        catch (final ParseException pe)
4008        {
4009          Debug.debugException(pe);
4010          throw new LDIFException(
4011               ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
4012                    firstLineNumber, pe.getMessage()),
4013               firstLineNumber, true, ldifLines, pe);
4014        }
4015        catch (final Exception e)
4016        {
4017          Debug.debugException(e);
4018          throw new LDIFException(
4019               ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
4020                    firstLineNumber, e),
4021               firstLineNumber, true, ldifLines, e);
4022        }
4023      }
4024      else
4025      {
4026        // Skip over any spaces leading up to the value, and then the rest of
4027        // the string is the new superior DN.
4028        int pos = colonPos+1;
4029        while ((pos < length) && (line.charAt(pos) == ' '))
4030        {
4031          pos++;
4032        }
4033
4034        newSuperiorDN = line.substring(pos);
4035      }
4036    }
4037    else
4038    {
4039      newSuperiorDN = null;
4040    }
4041
4042
4043    // There must not be any more lines.
4044    if (iterator.hasNext())
4045    {
4046      throw new LDIFException(ERR_READ_CR_EXTRA_MODDN_DATA.get(firstLineNumber),
4047                              firstLineNumber, true, ldifLines, null);
4048    }
4049
4050    return new LDIFModifyDNChangeRecord(dn, newRDN, deleteOldRDN,
4051         newSuperiorDN, controls);
4052  }
4053
4054
4055
4056  /**
4057   * Examines the line contained in the provided buffer to determine whether it
4058   * may contain one or more illegal trailing spaces.  If it does, then those
4059   * spaces will either be stripped out or an exception will be thrown to
4060   * indicate that they are illegal.
4061   *
4062   * @param  buffer                 The buffer to be examined.
4063   * @param  dn                     The DN of the LDIF record being parsed.  It
4064   *                                may be {@code null} if the DN is not yet
4065   *                                known (e.g., because the provided line is
4066   *                                expected to contain that DN).
4067   * @param  firstLineNumber        The approximate line number in the LDIF
4068   *                                source on which the LDIF record begins.
4069   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
4070   *                                encountering attribute values which are not
4071   *                                base64-encoded but contain trailing spaces.
4072   *
4073   * @throws  LDIFException  If the line contained in the provided buffer ends
4074   *                         with one or more illegal trailing spaces and
4075   *                         {@code stripTrailingSpaces} was provided with a
4076   *                         value of {@code false}.
4077   */
4078  private static void handleTrailingSpaces(final StringBuilder buffer,
4079                           final String dn, final long firstLineNumber,
4080                           final TrailingSpaceBehavior trailingSpaceBehavior)
4081          throws LDIFException
4082  {
4083    int pos = buffer.length() - 1;
4084    boolean trailingFound = false;
4085    while ((pos >= 0) && (buffer.charAt(pos) == ' '))
4086    {
4087      trailingFound = true;
4088      pos--;
4089    }
4090
4091    if (trailingFound && (buffer.charAt(pos) != ':'))
4092    {
4093      switch (trailingSpaceBehavior)
4094      {
4095        case STRIP:
4096          buffer.setLength(pos+1);
4097          break;
4098
4099        case REJECT:
4100          if (dn == null)
4101          {
4102            throw new LDIFException(
4103                 ERR_READ_ILLEGAL_TRAILING_SPACE_WITHOUT_DN.get(firstLineNumber,
4104                      buffer.toString()),
4105                 firstLineNumber, true);
4106          }
4107          else
4108          {
4109            throw new LDIFException(
4110                 ERR_READ_ILLEGAL_TRAILING_SPACE_WITH_DN.get(dn,
4111                      firstLineNumber, buffer.toString()),
4112                 firstLineNumber, true);
4113          }
4114
4115        case RETAIN:
4116        default:
4117          // No action will be taken.
4118          break;
4119      }
4120    }
4121  }
4122
4123
4124
4125  /**
4126   * This represents an unparsed LDIFRecord.  It stores the line number of the
4127   * first line of the record and each line of the record.
4128   */
4129  private static final class UnparsedLDIFRecord
4130  {
4131    private final ArrayList<StringBuilder> lineList;
4132    private final long firstLineNumber;
4133    private final Exception failureCause;
4134    private final boolean isEOF;
4135    private final DuplicateValueBehavior duplicateValueBehavior;
4136    private final Schema schema;
4137    private final TrailingSpaceBehavior trailingSpaceBehavior;
4138
4139
4140
4141    /**
4142     * Constructor.
4143     *
4144     * @param  lineList                The lines that comprise the LDIF record.
4145     * @param  duplicateValueBehavior  The behavior to exhibit if the entry
4146     *                                 contains duplicate attribute values.
4147     * @param  trailingSpaceBehavior   Specifies the behavior to exhibit when
4148     *                                 encountering trailing spaces in
4149     *                                 non-base64-encoded attribute values.
4150     * @param  schema                  The schema to use when parsing, if
4151     *                                 applicable.
4152     * @param  firstLineNumber         The first line number of the LDIF record.
4153     */
4154    private UnparsedLDIFRecord(final ArrayList<StringBuilder> lineList,
4155                 final DuplicateValueBehavior duplicateValueBehavior,
4156                 final TrailingSpaceBehavior trailingSpaceBehavior,
4157                 final Schema schema, final long firstLineNumber)
4158    {
4159      this.lineList               = lineList;
4160      this.firstLineNumber        = firstLineNumber;
4161      this.duplicateValueBehavior = duplicateValueBehavior;
4162      this.trailingSpaceBehavior  = trailingSpaceBehavior;
4163      this.schema                 = schema;
4164
4165      failureCause = null;
4166      isEOF =
4167           (firstLineNumber < 0) || ((lineList != null) && lineList.isEmpty());
4168    }
4169
4170
4171
4172    /**
4173     * Constructor.
4174     *
4175     * @param failureCause  The Exception thrown when reading from the input.
4176     */
4177    private UnparsedLDIFRecord(final Exception failureCause)
4178    {
4179      this.failureCause = failureCause;
4180
4181      lineList               = null;
4182      firstLineNumber        = 0;
4183      duplicateValueBehavior = DuplicateValueBehavior.REJECT;
4184      trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
4185      schema                 = null;
4186      isEOF                  = false;
4187    }
4188
4189
4190
4191    /**
4192     * Return the lines that comprise the LDIF record.
4193     *
4194     * @return  The lines that comprise the LDIF record.
4195     */
4196    private ArrayList<StringBuilder> getLineList()
4197    {
4198      return lineList;
4199    }
4200
4201
4202
4203    /**
4204     * Retrieves the behavior to exhibit when encountering duplicate attribute
4205     * values.
4206     *
4207     * @return  The behavior to exhibit when encountering duplicate attribute
4208     *          values.
4209     */
4210    private DuplicateValueBehavior getDuplicateValueBehavior()
4211    {
4212      return duplicateValueBehavior;
4213    }
4214
4215
4216
4217    /**
4218     * Retrieves the behavior that should be exhibited when encountering
4219     * attribute values which are not base64-encoded but contain trailing
4220     * spaces.  The LDIF specification strongly recommends that any value which
4221     * legitimately contains trailing spaces be base64-encoded, but the LDAP SDK
4222     * LDIF parser may be configured to automatically strip these spaces, to
4223     * preserve them, or to reject any entry or change record containing them.
4224     *
4225     * @return  The behavior that should be exhibited when encountering
4226     *          attribute values which are not base64-encoded but contain
4227     *          trailing spaces.
4228     */
4229    private TrailingSpaceBehavior getTrailingSpaceBehavior()
4230    {
4231      return trailingSpaceBehavior;
4232    }
4233
4234
4235
4236    /**
4237     * Retrieves the schema that should be used when parsing the record, if
4238     * applicable.
4239     *
4240     * @return  The schema that should be used when parsing the record, or
4241     *          {@code null} if none should be used.
4242     */
4243    private Schema getSchema()
4244    {
4245      return schema;
4246    }
4247
4248
4249
4250    /**
4251     * Return the first line number of the LDIF record.
4252     *
4253     * @return  The first line number of the LDIF record.
4254     */
4255    private long getFirstLineNumber()
4256    {
4257      return firstLineNumber;
4258    }
4259
4260
4261
4262    /**
4263     * Return {@code true} iff the end of the input was reached.
4264     *
4265     * @return  {@code true} iff the end of the input was reached.
4266     */
4267    private boolean isEOF()
4268    {
4269      return isEOF;
4270    }
4271
4272
4273
4274    /**
4275     * Returns the reason that reading the record lines failed.  This normally
4276     * is only non-null if something bad happened to the input stream (like
4277     * a disk read error).
4278     *
4279     * @return  The reason that reading the record lines failed.
4280     */
4281    private Exception getFailureCause()
4282    {
4283      return failureCause;
4284    }
4285  }
4286
4287
4288  /**
4289   * When processing in asynchronous mode, this thread is responsible for
4290   * reading the raw unparsed records from the input and submitting them for
4291   * processing.
4292   */
4293  private final class LineReaderThread
4294       extends Thread
4295  {
4296    /**
4297     * Constructor.
4298     */
4299    private LineReaderThread()
4300    {
4301      super("Asynchronous LDIF line reader");
4302      setDaemon(true);
4303    }
4304
4305
4306
4307    /**
4308     * Reads raw, unparsed records from the input and submits them for
4309     * processing until the input is finished or closed.
4310     */
4311    @Override()
4312    public void run()
4313    {
4314      try
4315      {
4316        boolean stopProcessing = false;
4317        while (!stopProcessing)
4318        {
4319          UnparsedLDIFRecord unparsedRecord;
4320          try
4321          {
4322            unparsedRecord = readUnparsedRecord();
4323          }
4324          catch (final IOException e)
4325          {
4326            Debug.debugException(e);
4327            unparsedRecord = new UnparsedLDIFRecord(e);
4328            stopProcessing = true;
4329          }
4330          catch (final Exception e)
4331          {
4332            Debug.debugException(e);
4333            unparsedRecord = new UnparsedLDIFRecord(e);
4334          }
4335
4336          try
4337          {
4338            asyncParser.submit(unparsedRecord);
4339          }
4340          catch (final InterruptedException e)
4341          {
4342            Debug.debugException(e);
4343            // If this thread is interrupted, then someone wants us to stop
4344            // processing, so that's what we'll do.
4345            Thread.currentThread().interrupt();
4346            stopProcessing = true;
4347          }
4348
4349          if ((unparsedRecord == null) || unparsedRecord.isEOF())
4350          {
4351            stopProcessing = true;
4352          }
4353        }
4354      }
4355      finally
4356      {
4357        try
4358        {
4359          asyncParser.shutdown();
4360        }
4361        catch (final InterruptedException e)
4362        {
4363          Debug.debugException(e);
4364          Thread.currentThread().interrupt();
4365        }
4366        finally
4367        {
4368          asyncParsingComplete.set(true);
4369        }
4370      }
4371    }
4372  }
4373
4374
4375
4376  /**
4377   * Used to parse Records asynchronously.
4378   */
4379  private final class RecordParser implements Processor<UnparsedLDIFRecord,
4380                                                        LDIFRecord>
4381  {
4382    /**
4383     * {@inheritDoc}
4384     */
4385    @Override()
4386    public LDIFRecord process(final UnparsedLDIFRecord input)
4387           throws LDIFException
4388    {
4389      LDIFRecord record = decodeRecord(input, relativeBasePath, schema);
4390
4391      if ((record instanceof Entry) && (entryTranslator != null))
4392      {
4393        record = entryTranslator.translate((Entry) record,
4394             input.getFirstLineNumber());
4395
4396        if (record == null)
4397        {
4398          record = SKIP_ENTRY;
4399        }
4400      }
4401      if ((record instanceof LDIFChangeRecord) &&
4402          (changeRecordTranslator != null))
4403      {
4404        record = changeRecordTranslator.translate((LDIFChangeRecord) record,
4405             input.getFirstLineNumber());
4406
4407        if (record == null)
4408        {
4409          record = SKIP_ENTRY;
4410        }
4411      }
4412      return record;
4413    }
4414  }
4415}