001/*
002 * Copyright 2016-2020 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2016-2020 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2016-2020 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldap.sdk.transformations;
037
038
039
040import java.io.File;
041import java.io.FileOutputStream;
042import java.io.InputStream;
043import java.io.OutputStream;
044import java.util.ArrayList;
045import java.util.EnumSet;
046import java.util.Iterator;
047import java.util.LinkedHashMap;
048import java.util.List;
049import java.util.Set;
050import java.util.TreeMap;
051import java.util.concurrent.atomic.AtomicLong;
052import java.util.zip.GZIPOutputStream;
053
054import com.unboundid.ldap.sdk.Attribute;
055import com.unboundid.ldap.sdk.ChangeType;
056import com.unboundid.ldap.sdk.DN;
057import com.unboundid.ldap.sdk.Entry;
058import com.unboundid.ldap.sdk.LDAPException;
059import com.unboundid.ldap.sdk.ResultCode;
060import com.unboundid.ldap.sdk.Version;
061import com.unboundid.ldap.sdk.schema.Schema;
062import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
063import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator;
064import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator;
065import com.unboundid.ldif.LDIFException;
066import com.unboundid.ldif.LDIFReader;
067import com.unboundid.ldif.LDIFReaderChangeRecordTranslator;
068import com.unboundid.ldif.LDIFReaderEntryTranslator;
069import com.unboundid.ldif.LDIFRecord;
070import com.unboundid.util.ByteStringBuffer;
071import com.unboundid.util.CommandLineTool;
072import com.unboundid.util.Debug;
073import com.unboundid.util.ObjectPair;
074import com.unboundid.util.PassphraseEncryptedOutputStream;
075import com.unboundid.util.StaticUtils;
076import com.unboundid.util.ThreadSafety;
077import com.unboundid.util.ThreadSafetyLevel;
078import com.unboundid.util.args.ArgumentException;
079import com.unboundid.util.args.ArgumentParser;
080import com.unboundid.util.args.BooleanArgument;
081import com.unboundid.util.args.DNArgument;
082import com.unboundid.util.args.FileArgument;
083import com.unboundid.util.args.FilterArgument;
084import com.unboundid.util.args.IntegerArgument;
085import com.unboundid.util.args.ScopeArgument;
086import com.unboundid.util.args.StringArgument;
087
088import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*;
089
090
091
092/**
093 * This class provides a command-line tool that can be used to apply a number of
094 * transformations to an LDIF file.  The transformations that can be applied
095 * include:
096 * <UL>
097 *   <LI>
098 *     It can scramble the values of a specified set of attributes in a manner
099 *     that attempts to preserve the syntax and consistently scrambles the same
100 *     value to the same representation.
101 *   </LI>
102 *   <LI>
103 *     It can strip a specified set of attributes out of entries.
104 *   </LI>
105 *   <LI>
106 *     It can redact the values of a specified set of attributes, to indicate
107 *     that the values are there but providing no information about what their
108 *     values are.
109 *   </LI>
110 *   <LI>
111 *     It can replace the values of a specified attribute with a given set of
112 *     values.
113 *   </LI>
114 *   <LI>
115 *     It can add an attribute with a given set of values to any entry that does
116 *     not contain that attribute.
117 *   </LI>
118 *   <LI>
119 *     It can replace the values of a specified attribute with a value that
120 *     contains a sequentially-incrementing counter.
121 *   </LI>
122 *   <LI>
123 *     It can strip entries matching a given base DN, scope, and filter out of
124 *     the LDIF file.
125 *   </LI>
126 *   <LI>
127 *     It can perform DN mapping, so that entries that exist below one base DN
128 *     are moved below a different base DN.
129 *   </LI>
130 *   <LI>
131 *     It can perform attribute mapping, to replace uses of one attribute name
132 *     with another.
133 *   </LI>
134 * </UL>
135 */
136@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
137public final class TransformLDIF
138       extends CommandLineTool
139       implements LDIFReaderEntryTranslator
140{
141  /**
142   * The maximum length of any message to write to standard output or standard
143   * error.
144   */
145  private static final int MAX_OUTPUT_LINE_LENGTH =
146       StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
147
148
149
150  // The arguments for use by this program.
151  private BooleanArgument addToExistingValues = null;
152  private BooleanArgument appendToTargetLDIF = null;
153  private BooleanArgument compressTarget = null;
154  private BooleanArgument encryptTarget = null;
155  private BooleanArgument excludeRecordsWithoutChangeType = null;
156  private BooleanArgument excludeNonMatchingEntries = null;
157  private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null;
158  private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null;
159  private BooleanArgument hideRedactedValueCount = null;
160  private BooleanArgument processDNs = null;
161  private BooleanArgument sourceCompressed = null;
162  private BooleanArgument sourceContainsChangeRecords = null;
163  private BooleanArgument sourceFromStandardInput = null;
164  private BooleanArgument targetToStandardOutput = null;
165  private DNArgument addAttributeBaseDN = null;
166  private DNArgument excludeEntryBaseDN = null;
167  private DNArgument flattenBaseDN = null;
168  private DNArgument moveSubtreeFrom = null;
169  private DNArgument moveSubtreeTo = null;
170  private FileArgument encryptionPassphraseFile = null;
171  private FileArgument schemaPath = null;
172  private FileArgument sourceLDIF = null;
173  private FileArgument targetLDIF = null;
174  private FilterArgument addAttributeFilter = null;
175  private FilterArgument excludeEntryFilter = null;
176  private FilterArgument flattenExcludeFilter = null;
177  private IntegerArgument initialSequentialValue = null;
178  private IntegerArgument numThreads = null;
179  private IntegerArgument randomSeed = null;
180  private IntegerArgument sequentialValueIncrement = null;
181  private IntegerArgument wrapColumn = null;
182  private ScopeArgument addAttributeScope = null;
183  private ScopeArgument excludeEntryScope = null;
184  private StringArgument addAttributeName = null;
185  private StringArgument addAttributeValue = null;
186  private StringArgument excludeAttribute = null;
187  private StringArgument excludeChangeType  = null;
188  private StringArgument redactAttribute = null;
189  private StringArgument renameAttributeFrom = null;
190  private StringArgument renameAttributeTo = null;
191  private StringArgument replaceValuesAttribute = null;
192  private StringArgument replacementValue = null;
193  private StringArgument scrambleAttribute = null;
194  private StringArgument scrambleJSONField = null;
195  private StringArgument sequentialAttribute = null;
196  private StringArgument textAfterSequentialValue = null;
197  private StringArgument textBeforeSequentialValue = null;
198
199  // A set of thread-local byte stream buffers that will be used to construct
200  // the LDIF representations of records.
201  private final ThreadLocal<ByteStringBuffer> byteStringBuffers =
202       new ThreadLocal<>();
203
204
205
206  /**
207   * Invokes this tool with the provided set of arguments.
208   *
209   * @param  args  The command-line arguments provided to this program.
210   */
211  public static void main(final String... args)
212  {
213    final ResultCode resultCode = main(System.out, System.err, args);
214    if (resultCode != ResultCode.SUCCESS)
215    {
216      System.exit(resultCode.intValue());
217    }
218  }
219
220
221
222  /**
223   * Invokes this tool with the provided set of arguments.
224   *
225   * @param  out   The output stream to use for standard output.  It may be
226   *               {@code null} if standard output should be suppressed.
227   * @param  err   The output stream to use for standard error.  It may be
228   *               {@code null} if standard error should be suppressed.
229   * @param  args  The command-line arguments provided to this program.
230   *
231   * @return  A result code indicating whether processing completed
232   *          successfully.
233   */
234  public static ResultCode main(final OutputStream out, final OutputStream err,
235                                final String... args)
236  {
237    final TransformLDIF tool = new TransformLDIF(out, err);
238    return tool.runTool(args);
239  }
240
241
242
243  /**
244   * Creates a new instance of this tool with the provided information.
245   *
246   * @param  out  The output stream to use for standard output.  It may be
247   *              {@code null} if standard output should be suppressed.
248   * @param  err  The output stream to use for standard error.  It may be
249   *              {@code null} if standard error should be suppressed.
250   */
251  public TransformLDIF(final OutputStream out, final OutputStream err)
252  {
253    super(out, err);
254  }
255
256
257
258  /**
259   * {@inheritDoc}
260   */
261  @Override()
262  public String getToolName()
263  {
264    return "transform-ldif";
265  }
266
267
268
269  /**
270   * {@inheritDoc}
271   */
272  @Override()
273  public String getToolDescription()
274  {
275    return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get();
276  }
277
278
279
280  /**
281   * {@inheritDoc}
282   */
283  @Override()
284  public String getToolVersion()
285  {
286    return Version.NUMERIC_VERSION_STRING;
287  }
288
289
290
291  /**
292   * {@inheritDoc}
293   */
294  @Override()
295  public boolean supportsInteractiveMode()
296  {
297    return true;
298  }
299
300
301
302  /**
303   * {@inheritDoc}
304   */
305  @Override()
306  public boolean defaultsToInteractiveMode()
307  {
308    return true;
309  }
310
311
312
313  /**
314   * {@inheritDoc}
315   */
316  @Override()
317  public boolean supportsPropertiesFile()
318  {
319    return true;
320  }
321
322
323
324  /**
325   * {@inheritDoc}
326   */
327  @Override()
328  public void addToolArguments(final ArgumentParser parser)
329         throws ArgumentException
330  {
331    // Add arguments pertaining to the source and target LDIF files.
332    sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null,
333         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true,
334         false);
335    sourceLDIF.addLongIdentifier("inputLDIF", true);
336    sourceLDIF.addLongIdentifier("source-ldif", true);
337    sourceLDIF.addLongIdentifier("input-ldif", true);
338    sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
339    parser.addArgument(sourceLDIF);
340
341    sourceFromStandardInput = new BooleanArgument(null,
342         "sourceFromStandardInput", 1,
343         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get());
344    sourceFromStandardInput.addLongIdentifier("source-from-standard-input",
345         true);
346    sourceFromStandardInput.setArgumentGroupName(
347         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
348    parser.addArgument(sourceFromStandardInput);
349    parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput);
350    parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput);
351
352    targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null,
353         INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true,
354         false);
355    targetLDIF.addLongIdentifier("outputLDIF", true);
356    targetLDIF.addLongIdentifier("target-ldif", true);
357    targetLDIF.addLongIdentifier("output-ldif", true);
358    targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
359    parser.addArgument(targetLDIF);
360
361    targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput",
362         1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get());
363    targetToStandardOutput.addLongIdentifier("target-to-standard-output", true);
364    targetToStandardOutput.setArgumentGroupName(
365         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
366    parser.addArgument(targetToStandardOutput);
367    parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput);
368
369    sourceContainsChangeRecords = new BooleanArgument(null,
370         "sourceContainsChangeRecords",
371         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get());
372    sourceContainsChangeRecords.addLongIdentifier(
373         "source-contains-change-records", true);
374    sourceContainsChangeRecords.setArgumentGroupName(
375         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
376    parser.addArgument(sourceContainsChangeRecords);
377
378    appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF",
379         INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get());
380    appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true);
381    appendToTargetLDIF.setArgumentGroupName(
382         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
383    parser.addArgument(appendToTargetLDIF);
384    parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF);
385
386    wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null,
387         INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE);
388    wrapColumn.addLongIdentifier("wrap-column", true);
389    wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
390    parser.addArgument(wrapColumn);
391
392    sourceCompressed = new BooleanArgument('C', "sourceCompressed",
393         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get());
394    sourceCompressed.addLongIdentifier("inputCompressed", true);
395    sourceCompressed.addLongIdentifier("source-compressed", true);
396    sourceCompressed.addLongIdentifier("input-compressed", true);
397    sourceCompressed.setArgumentGroupName(
398         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
399    parser.addArgument(sourceCompressed);
400
401    compressTarget = new BooleanArgument('c', "compressTarget",
402         INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get());
403    compressTarget.addLongIdentifier("compressOutput", true);
404    compressTarget.addLongIdentifier("compress", true);
405    compressTarget.addLongIdentifier("compress-target", true);
406    compressTarget.addLongIdentifier("compress-output", true);
407    compressTarget.setArgumentGroupName(
408         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
409    parser.addArgument(compressTarget);
410
411    encryptTarget = new BooleanArgument(null, "encryptTarget",
412         INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get());
413    encryptTarget.addLongIdentifier("encryptOutput", true);
414    encryptTarget.addLongIdentifier("encrypt", true);
415    encryptTarget.addLongIdentifier("encrypt-target", true);
416    encryptTarget.addLongIdentifier("encrypt-output", true);
417    encryptTarget.setArgumentGroupName(
418         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
419    parser.addArgument(encryptTarget);
420
421    encryptionPassphraseFile = new FileArgument(null,
422         "encryptionPassphraseFile", false, 1, null,
423         INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true,
424         true, false);
425    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
426    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
427         true);
428    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
429         true);
430    encryptionPassphraseFile.setArgumentGroupName(
431         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
432    parser.addArgument(encryptionPassphraseFile);
433
434
435    // Add arguments pertaining to attribute scrambling.
436    scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0,
437         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
438         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get());
439    scrambleAttribute.addLongIdentifier("attributeName", true);
440    scrambleAttribute.addLongIdentifier("scramble-attribute", true);
441    scrambleAttribute.addLongIdentifier("attribute-name", true);
442    scrambleAttribute.setArgumentGroupName(
443         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
444    parser.addArgument(scrambleAttribute);
445
446    scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0,
447         INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(),
448         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get(
449              scrambleAttribute.getIdentifierString()));
450    scrambleJSONField.addLongIdentifier("scramble-json-field", true);
451    scrambleJSONField.setArgumentGroupName(
452         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
453    parser.addArgument(scrambleJSONField);
454    parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute);
455
456    randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null,
457         INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get());
458    randomSeed.addLongIdentifier("random-seed", true);
459    randomSeed.setArgumentGroupName(
460         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
461    parser.addArgument(randomSeed);
462
463
464    // Add arguments pertaining to replacing attribute values with a generated
465    // value using a sequential counter.
466    sequentialAttribute = new StringArgument('S', "sequentialAttribute",
467         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
468         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get(
469              sourceContainsChangeRecords.getIdentifierString()));
470    sequentialAttribute.addLongIdentifier("sequentialAttributeName", true);
471    sequentialAttribute.addLongIdentifier("sequential-attribute", true);
472    sequentialAttribute.addLongIdentifier("sequential-attribute-name", true);
473    sequentialAttribute.setArgumentGroupName(
474         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
475    parser.addArgument(sequentialAttribute);
476    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
477         sequentialAttribute);
478
479    initialSequentialValue = new IntegerArgument('i', "initialSequentialValue",
480         false, 1, null,
481         INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get(
482              sequentialAttribute.getIdentifierString()));
483    initialSequentialValue.addLongIdentifier("initial-sequential-value", true);
484    initialSequentialValue.setArgumentGroupName(
485         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
486    parser.addArgument(initialSequentialValue);
487    parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute);
488
489    sequentialValueIncrement = new IntegerArgument(null,
490         "sequentialValueIncrement", false, 1, null,
491         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get(
492              sequentialAttribute.getIdentifierString()));
493    sequentialValueIncrement.addLongIdentifier("sequential-value-increment",
494         true);
495    sequentialValueIncrement.setArgumentGroupName(
496         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
497    parser.addArgument(sequentialValueIncrement);
498    parser.addDependentArgumentSet(sequentialValueIncrement,
499         sequentialAttribute);
500
501    textBeforeSequentialValue = new StringArgument(null,
502         "textBeforeSequentialValue", false, 1, null,
503         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get(
504              sequentialAttribute.getIdentifierString()));
505    textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value",
506         true);
507    textBeforeSequentialValue.setArgumentGroupName(
508         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
509    parser.addArgument(textBeforeSequentialValue);
510    parser.addDependentArgumentSet(textBeforeSequentialValue,
511         sequentialAttribute);
512
513    textAfterSequentialValue = new StringArgument(null,
514         "textAfterSequentialValue", false, 1, null,
515         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get(
516              sequentialAttribute.getIdentifierString()));
517    textAfterSequentialValue.addLongIdentifier("text-after-sequential-value",
518         true);
519    textAfterSequentialValue.setArgumentGroupName(
520         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
521    parser.addArgument(textAfterSequentialValue);
522    parser.addDependentArgumentSet(textAfterSequentialValue,
523         sequentialAttribute);
524
525
526    // Add arguments pertaining to attribute value replacement.
527    replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute",
528         false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
529         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get(
530              sourceContainsChangeRecords.getIdentifierString()));
531    replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true);
532    replaceValuesAttribute.setArgumentGroupName(
533         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
534    parser.addArgument(replaceValuesAttribute);
535    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
536         replaceValuesAttribute);
537
538    replacementValue = new StringArgument(null, "replacementValue", false, 0,
539         null,
540         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get(
541              replaceValuesAttribute.getIdentifierString()));
542    replacementValue.addLongIdentifier("replacement-value", true);
543    replacementValue.setArgumentGroupName(
544         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
545    parser.addArgument(replacementValue);
546    parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue);
547    parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute);
548
549
550    // Add arguments pertaining to adding missing attributes.
551    addAttributeName = new StringArgument(null, "addAttributeName", false, 1,
552         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
553         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get(
554              "--addAttributeValue",
555              sourceContainsChangeRecords.getIdentifierString()));
556    addAttributeName.addLongIdentifier("add-attribute-name", true);
557    addAttributeName.setArgumentGroupName(
558         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
559    parser.addArgument(addAttributeName);
560    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
561         addAttributeName);
562
563    addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0,
564         null,
565         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get(
566              addAttributeName.getIdentifierString()));
567    addAttributeValue.addLongIdentifier("add-attribute-value", true);
568    addAttributeValue.setArgumentGroupName(
569         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
570    parser.addArgument(addAttributeValue);
571    parser.addDependentArgumentSet(addAttributeName, addAttributeValue);
572    parser.addDependentArgumentSet(addAttributeValue, addAttributeName);
573
574    addToExistingValues = new BooleanArgument(null, "addToExistingValues",
575         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get(
576              addAttributeName.getIdentifierString(),
577              addAttributeValue.getIdentifierString()));
578    addToExistingValues.addLongIdentifier("add-to-existing-values", true);
579    addToExistingValues.setArgumentGroupName(
580         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
581    parser.addArgument(addToExistingValues);
582    parser.addDependentArgumentSet(addToExistingValues, addAttributeName);
583
584    addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1,
585         null,
586         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get(
587              addAttributeName.getIdentifierString()));
588    addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true);
589    addAttributeBaseDN.setArgumentGroupName(
590         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
591    parser.addArgument(addAttributeBaseDN);
592    parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName);
593
594    addAttributeScope = new ScopeArgument(null, "addAttributeScope", false,
595         null,
596         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get(
597              addAttributeBaseDN.getIdentifierString(),
598              addAttributeName.getIdentifierString()));
599    addAttributeScope.addLongIdentifier("add-attribute-scope", true);
600    addAttributeScope.setArgumentGroupName(
601         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
602    parser.addArgument(addAttributeScope);
603    parser.addDependentArgumentSet(addAttributeScope, addAttributeName);
604
605    addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false,
606         1, null,
607         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get(
608              addAttributeName.getIdentifierString()));
609    addAttributeFilter.addLongIdentifier("add-attribute-filter", true);
610    addAttributeFilter.setArgumentGroupName(
611         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
612    parser.addArgument(addAttributeFilter);
613    parser.addDependentArgumentSet(addAttributeFilter, addAttributeName);
614
615
616    // Add arguments pertaining to renaming attributes.
617    renameAttributeFrom = new StringArgument(null, "renameAttributeFrom",
618         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
619         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get());
620    renameAttributeFrom.addLongIdentifier("rename-attribute-from", true);
621    renameAttributeFrom.setArgumentGroupName(
622         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
623    parser.addArgument(renameAttributeFrom);
624
625    renameAttributeTo = new StringArgument(null, "renameAttributeTo",
626         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
627         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get(
628              renameAttributeFrom.getIdentifierString()));
629    renameAttributeTo.addLongIdentifier("rename-attribute-to", true);
630    renameAttributeTo.setArgumentGroupName(
631         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
632    parser.addArgument(renameAttributeTo);
633    parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo);
634    parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom);
635
636
637    // Add arguments pertaining to flattening subtrees.
638    flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null,
639         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get());
640    flattenBaseDN.addLongIdentifier("flatten-base-dn", true);
641    flattenBaseDN.setArgumentGroupName(
642         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
643    parser.addArgument(flattenBaseDN);
644    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
645         flattenBaseDN);
646
647    flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null,
648         "flattenAddOmittedRDNAttributesToEntry", 1,
649         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get());
650    flattenAddOmittedRDNAttributesToEntry.addLongIdentifier(
651         "flatten-add-omitted-rdn-attributes-to-entry", true);
652    flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName(
653         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
654    parser.addArgument(flattenAddOmittedRDNAttributesToEntry);
655    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry,
656         flattenBaseDN);
657
658    flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null,
659         "flattenAddOmittedRDNAttributesToRDN", 1,
660         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get());
661    flattenAddOmittedRDNAttributesToRDN.addLongIdentifier(
662         "flatten-add-omitted-rdn-attributes-to-rdn", true);
663    flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName(
664         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
665    parser.addArgument(flattenAddOmittedRDNAttributesToRDN);
666    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN,
667         flattenBaseDN);
668
669    flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter",
670         false, 1, null,
671         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get());
672    flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true);
673    flattenExcludeFilter.setArgumentGroupName(
674         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
675    parser.addArgument(flattenExcludeFilter);
676    parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN);
677
678
679    // Add arguments pertaining to moving subtrees.
680    moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null,
681         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get());
682    moveSubtreeFrom.addLongIdentifier("move-subtree-from", true);
683    moveSubtreeFrom.setArgumentGroupName(
684         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
685    parser.addArgument(moveSubtreeFrom);
686
687    moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null,
688         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get(
689              moveSubtreeFrom.getIdentifierString()));
690    moveSubtreeTo.addLongIdentifier("move-subtree-to", true);
691    moveSubtreeTo.setArgumentGroupName(
692         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
693    parser.addArgument(moveSubtreeTo);
694    parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo);
695    parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom);
696
697
698    // Add arguments pertaining to redacting attribute values.
699    redactAttribute = new StringArgument(null, "redactAttribute", false, 0,
700         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
701         INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get());
702    redactAttribute.addLongIdentifier("redact-attribute", true);
703    redactAttribute.setArgumentGroupName(
704         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
705    parser.addArgument(redactAttribute);
706
707    hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount",
708         INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get());
709    hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count",
710         true);
711    hideRedactedValueCount.setArgumentGroupName(
712         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
713    parser.addArgument(hideRedactedValueCount);
714    parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute);
715
716
717    // Add arguments pertaining to excluding attributes and entries.
718    excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0,
719         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
720         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get());
721    excludeAttribute.addLongIdentifier("suppressAttribute", true);
722    excludeAttribute.addLongIdentifier("exclude-attribute", true);
723    excludeAttribute.addLongIdentifier("suppress-attribute", true);
724    excludeAttribute.setArgumentGroupName(
725         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
726    parser.addArgument(excludeAttribute);
727
728    excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1,
729         null,
730         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get(
731              sourceContainsChangeRecords.getIdentifierString()));
732    excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true);
733    excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true);
734    excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true);
735    excludeEntryBaseDN.setArgumentGroupName(
736         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
737    parser.addArgument(excludeEntryBaseDN);
738    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
739         excludeEntryBaseDN);
740
741    excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false,
742         null,
743         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get(
744              sourceContainsChangeRecords.getIdentifierString()));
745    excludeEntryScope.addLongIdentifier("suppressEntryScope", true);
746    excludeEntryScope.addLongIdentifier("exclude-entry-scope", true);
747    excludeEntryScope.addLongIdentifier("suppress-entry-scope", true);
748    excludeEntryScope.setArgumentGroupName(
749         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
750    parser.addArgument(excludeEntryScope);
751    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
752         excludeEntryScope);
753
754    excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false,
755         1, null,
756         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get(
757              sourceContainsChangeRecords.getIdentifierString()));
758    excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true);
759    excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true);
760    excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true);
761    excludeEntryFilter.setArgumentGroupName(
762         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
763    parser.addArgument(excludeEntryFilter);
764    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
765         excludeEntryFilter);
766
767    excludeNonMatchingEntries = new BooleanArgument(null,
768         "excludeNonMatchingEntries",
769         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get());
770    excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries",
771         true);
772    excludeNonMatchingEntries.setArgumentGroupName(
773         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
774    parser.addArgument(excludeNonMatchingEntries);
775    parser.addDependentArgumentSet(excludeNonMatchingEntries,
776         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
777
778
779    // Add arguments for excluding records based on their change types.
780    excludeChangeType = new StringArgument(null, "excludeChangeType",
781         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_CHANGE_TYPES.get(),
782         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_CHANGE_TYPE.get(),
783         StaticUtils.setOf("add", "delete", "modify", "moddn"));
784    excludeChangeType.addLongIdentifier("exclude-change-type", true);
785    excludeChangeType.addLongIdentifier("exclude-changetype", true);
786    excludeChangeType.setArgumentGroupName(
787         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
788    parser.addArgument(excludeChangeType);
789
790
791    // Add arguments for excluding records that don't have a change type.
792    excludeRecordsWithoutChangeType = new BooleanArgument(null,
793         "excludeRecordsWithoutChangeType", 1,
794         INFO_TRANSFORM_LDIF_EXCLUDE_WITHOUT_CHANGETYPE.get());
795    excludeRecordsWithoutChangeType.addLongIdentifier(
796         "exclude-records-without-change-type", true);
797    excludeRecordsWithoutChangeType.addLongIdentifier(
798         "exclude-records-without-changetype", true);
799    excludeRecordsWithoutChangeType.setArgumentGroupName(
800         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
801    parser.addArgument(excludeRecordsWithoutChangeType);
802
803
804    // Add the remaining arguments.
805    schemaPath = new FileArgument(null, "schemaPath", false, 0, null,
806         INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(),
807         true, true, false, false);
808    schemaPath.addLongIdentifier("schemaFile", true);
809    schemaPath.addLongIdentifier("schemaDirectory", true);
810    schemaPath.addLongIdentifier("schema-path", true);
811    schemaPath.addLongIdentifier("schema-file", true);
812    schemaPath.addLongIdentifier("schema-directory", true);
813    parser.addArgument(schemaPath);
814
815    numThreads = new IntegerArgument('t', "numThreads", false, 1, null,
816         INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE,
817         1);
818    numThreads.addLongIdentifier("num-threads", true);
819    parser.addArgument(numThreads);
820
821    processDNs = new BooleanArgument('d', "processDNs",
822         INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get());
823    processDNs.addLongIdentifier("process-dns", true);
824    parser.addArgument(processDNs);
825
826
827    // Ensure that at least one kind of transformation was requested.
828    parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute,
829         replaceValuesAttribute, addAttributeName, renameAttributeFrom,
830         flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute,
831         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter,
832         excludeChangeType, excludeRecordsWithoutChangeType);
833  }
834
835
836
837  /**
838   * {@inheritDoc}
839   */
840  @Override()
841  public void doExtendedArgumentValidation()
842         throws ArgumentException
843  {
844    // Ideally, exactly one of the targetLDIF and targetToStandardOutput
845    // arguments should always be provided.  But in order to preserve backward
846    // compatibility with a legacy scramble-ldif tool, we will allow both to be
847    // omitted if either --scrambleAttribute or --sequentialArgument is
848    // provided.  In that case, the path of the output file will be the path of
849    // the first input file with ".scrambled" appended to it.
850    if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent()))
851    {
852      if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent()))
853      {
854        throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get(
855             targetLDIF.getIdentifierString(),
856             targetToStandardOutput.getIdentifierString()));
857      }
858    }
859
860
861    // Make sure that the --renameAttributeFrom and --renameAttributeTo
862    // arguments were provided an equal number of times.
863    final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences();
864    final int renameToOccurrences = renameAttributeTo.getNumOccurrences();
865    if (renameFromOccurrences != renameToOccurrences)
866    {
867      throw new ArgumentException(
868           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
869                renameAttributeFrom.getIdentifierString(),
870                renameAttributeTo.getIdentifierString()));
871    }
872
873
874    // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were
875    // provided an equal number of times.
876    final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences();
877    final int moveToOccurrences = moveSubtreeTo.getNumOccurrences();
878    if (moveFromOccurrences != moveToOccurrences)
879    {
880      throw new ArgumentException(
881           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
882                moveSubtreeFrom.getIdentifierString(),
883                moveSubtreeTo.getIdentifierString()));
884    }
885  }
886
887
888
889  /**
890   * {@inheritDoc}
891   */
892  @Override()
893  public ResultCode doToolProcessing()
894  {
895    final Schema schema;
896    try
897    {
898      schema = getSchema();
899    }
900    catch (final LDAPException le)
901    {
902      wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage());
903      return le.getResultCode();
904    }
905
906
907    // If an encryption passphrase file is provided, then get the passphrase
908    // from it.
909    String encryptionPassphrase = null;
910    if (encryptionPassphraseFile.isPresent())
911    {
912      try
913      {
914        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
915             encryptionPassphraseFile.getValue());
916      }
917      catch (final LDAPException e)
918      {
919        wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage());
920        return e.getResultCode();
921      }
922    }
923
924
925    // Create the translators to use to apply the transformations.
926    final ArrayList<LDIFReaderEntryTranslator> entryTranslators =
927         new ArrayList<>(10);
928    final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators =
929         new ArrayList<>(10);
930
931    final AtomicLong excludedEntryCount = new AtomicLong(0L);
932    createTranslators(entryTranslators, changeRecordTranslators,
933         schema, excludedEntryCount);
934
935    final AggregateLDIFReaderEntryTranslator entryTranslator =
936         new AggregateLDIFReaderEntryTranslator(entryTranslators);
937    final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator =
938         new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators);
939
940
941    // Determine the path to the target file to be written.
942    final File targetFile;
943    if (targetLDIF.isPresent())
944    {
945      targetFile = targetLDIF.getValue();
946    }
947    else if (targetToStandardOutput.isPresent())
948    {
949      targetFile = null;
950    }
951    else
952    {
953      targetFile =
954           new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled");
955    }
956
957
958    // Create the LDIF reader.
959    final LDIFReader ldifReader;
960    try
961    {
962      final InputStream inputStream;
963      if (sourceLDIF.isPresent())
964      {
965        final ObjectPair<InputStream,String> p =
966             ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(),
967                  encryptionPassphrase, getOut(), getErr());
968        inputStream = p.getFirst();
969        if ((encryptionPassphrase == null) && (p.getSecond() != null))
970        {
971          encryptionPassphrase = p.getSecond();
972        }
973      }
974      else
975      {
976        inputStream = System.in;
977      }
978
979      ldifReader = new LDIFReader(inputStream, numThreads.getValue(),
980           entryTranslator, changeRecordTranslator);
981      if (schema != null)
982      {
983        ldifReader.setSchema(schema);
984      }
985    }
986    catch (final Exception e)
987    {
988      Debug.debugException(e);
989      wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
990           ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get(
991                StaticUtils.getExceptionMessage(e)));
992      return ResultCode.LOCAL_ERROR;
993    }
994
995
996    ResultCode resultCode = ResultCode.SUCCESS;
997    OutputStream outputStream = null;
998processingBlock:
999    try
1000    {
1001      // Create the output stream to use to write the transformed data.
1002      try
1003      {
1004        if (targetFile == null)
1005        {
1006          outputStream = getOut();
1007        }
1008        else
1009        {
1010          outputStream =
1011               new FileOutputStream(targetFile, appendToTargetLDIF.isPresent());
1012        }
1013
1014        if (encryptTarget.isPresent())
1015        {
1016          if (encryptionPassphrase == null)
1017          {
1018            encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase(
1019                 false, true, getOut(), getErr());
1020          }
1021
1022          outputStream = new PassphraseEncryptedOutputStream(
1023               encryptionPassphrase, outputStream);
1024        }
1025
1026        if (compressTarget.isPresent())
1027        {
1028          outputStream = new GZIPOutputStream(outputStream);
1029        }
1030      }
1031      catch (final Exception e)
1032      {
1033        Debug.debugException(e);
1034        wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1035             ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get(
1036                  targetFile.getAbsolutePath(),
1037                  StaticUtils.getExceptionMessage(e)));
1038        resultCode = ResultCode.LOCAL_ERROR;
1039        break processingBlock;
1040      }
1041
1042
1043      // Read the source data one record at a time.  The transformations will
1044      // automatically be applied by the LDIF reader's translators, and even if
1045      // there are multiple reader threads, we're guaranteed to get the results
1046      // in the right order.
1047      long entriesWritten = 0L;
1048      while (true)
1049      {
1050        final LDIFRecord ldifRecord;
1051        try
1052        {
1053          ldifRecord = ldifReader.readLDIFRecord();
1054        }
1055        catch (final LDIFException le)
1056        {
1057          Debug.debugException(le);
1058          if (le.mayContinueReading())
1059          {
1060            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1061                 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get(
1062                      StaticUtils.getExceptionMessage(le)));
1063            if (resultCode == ResultCode.SUCCESS)
1064            {
1065              resultCode = ResultCode.PARAM_ERROR;
1066            }
1067            continue;
1068          }
1069          else
1070          {
1071            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1072                 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get(
1073                      StaticUtils.getExceptionMessage(le)));
1074            if (resultCode == ResultCode.SUCCESS)
1075            {
1076              resultCode = ResultCode.PARAM_ERROR;
1077            }
1078            break processingBlock;
1079          }
1080        }
1081        catch (final Exception e)
1082        {
1083          Debug.debugException(e);
1084          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1085               ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get(
1086                    StaticUtils.getExceptionMessage(e)));
1087          resultCode = ResultCode.LOCAL_ERROR;
1088          break processingBlock;
1089        }
1090
1091
1092        // If the LDIF record is null, then we've run out of records so we're
1093        // done.
1094        if (ldifRecord == null)
1095        {
1096          break;
1097        }
1098
1099
1100        // Write the record to the output stream.
1101        try
1102        {
1103          if (ldifRecord instanceof PreEncodedLDIFEntry)
1104          {
1105            outputStream.write(
1106                 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes());
1107          }
1108          else
1109          {
1110            final ByteStringBuffer buffer = getBuffer();
1111            if (wrapColumn.isPresent())
1112            {
1113              ldifRecord.toLDIF(buffer, wrapColumn.getValue());
1114            }
1115            else
1116            {
1117              ldifRecord.toLDIF(buffer, 0);
1118            }
1119            buffer.append(StaticUtils.EOL_BYTES);
1120            buffer.write(outputStream);
1121          }
1122        }
1123        catch (final Exception e)
1124        {
1125          Debug.debugException(e);
1126          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1127               ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(),
1128                    StaticUtils.getExceptionMessage(e)));
1129          resultCode = ResultCode.LOCAL_ERROR;
1130          break processingBlock;
1131        }
1132
1133
1134        // If we've written a multiple of 1000 entries, print a progress
1135        // message.
1136        entriesWritten++;
1137        if ((! targetToStandardOutput.isPresent()) &&
1138            ((entriesWritten % 1000L) == 0))
1139        {
1140          final long numExcluded = excludedEntryCount.get();
1141          if (numExcluded > 0L)
1142          {
1143            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1144                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get(
1145                      entriesWritten, numExcluded));
1146          }
1147          else
1148          {
1149            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1150                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get(
1151                      entriesWritten));
1152          }
1153        }
1154      }
1155
1156
1157      if (! targetToStandardOutput.isPresent())
1158      {
1159        final long numExcluded = excludedEntryCount.get();
1160        if (numExcluded > 0L)
1161        {
1162          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1163               INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten,
1164                    numExcluded));
1165        }
1166        else
1167        {
1168          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1169               INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten));
1170        }
1171      }
1172    }
1173    finally
1174    {
1175      if (outputStream != null)
1176      {
1177        try
1178        {
1179          outputStream.close();
1180        }
1181        catch (final Exception e)
1182        {
1183          Debug.debugException(e);
1184          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1185               ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get(
1186                    targetFile.getAbsolutePath(),
1187                    StaticUtils.getExceptionMessage(e)));
1188          if (resultCode == ResultCode.SUCCESS)
1189          {
1190            resultCode = ResultCode.LOCAL_ERROR;
1191          }
1192        }
1193      }
1194
1195      try
1196      {
1197        ldifReader.close();
1198      }
1199      catch (final Exception e)
1200      {
1201        Debug.debugException(e);
1202        // We can ignore this.
1203      }
1204    }
1205
1206
1207    return resultCode;
1208  }
1209
1210
1211
1212  /**
1213   * Retrieves the schema that should be used for processing.
1214   *
1215   * @return  The schema that was created.
1216   *
1217   * @throws  LDAPException  If a problem is encountered while retrieving the
1218   *                         schema.
1219   */
1220  private Schema getSchema()
1221          throws LDAPException
1222  {
1223    // If any schema paths were specified, then load the schema only from those
1224    // paths.
1225    if (schemaPath.isPresent())
1226    {
1227      final ArrayList<File> schemaFiles = new ArrayList<>(10);
1228      for (final File path : schemaPath.getValues())
1229      {
1230        if (path.isFile())
1231        {
1232          schemaFiles.add(path);
1233        }
1234        else
1235        {
1236          final TreeMap<String,File> fileMap = new TreeMap<>();
1237          for (final File schemaDirFile : path.listFiles())
1238          {
1239            final String name = schemaDirFile.getName();
1240            if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif"))
1241            {
1242              fileMap.put(name, schemaDirFile);
1243            }
1244          }
1245          schemaFiles.addAll(fileMap.values());
1246        }
1247      }
1248
1249      if (schemaFiles.isEmpty())
1250      {
1251        throw new LDAPException(ResultCode.PARAM_ERROR,
1252             ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get(
1253                  schemaPath.getIdentifierString()));
1254      }
1255      else
1256      {
1257        try
1258        {
1259          return Schema.getSchema(schemaFiles);
1260        }
1261        catch (final Exception e)
1262        {
1263          Debug.debugException(e);
1264          throw new LDAPException(ResultCode.LOCAL_ERROR,
1265               ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get(
1266                    StaticUtils.getExceptionMessage(e)));
1267        }
1268      }
1269    }
1270    else
1271    {
1272      // If the INSTANCE_ROOT environment variable is set and it refers to a
1273      // directory that has a config/schema subdirectory that has one or more
1274      // schema files in it, then read the schema from that directory.
1275      try
1276      {
1277        final String instanceRootStr =
1278             StaticUtils.getEnvironmentVariable("INSTANCE_ROOT");
1279        if (instanceRootStr != null)
1280        {
1281          final File instanceRoot = new File(instanceRootStr);
1282          final File configDir = new File(instanceRoot, "config");
1283          final File schemaDir = new File(configDir, "schema");
1284          if (schemaDir.exists())
1285          {
1286            final TreeMap<String,File> fileMap = new TreeMap<>();
1287            for (final File schemaDirFile : schemaDir.listFiles())
1288            {
1289              final String name = schemaDirFile.getName();
1290              if (schemaDirFile.isFile() &&
1291                  name.toLowerCase().endsWith(".ldif"))
1292              {
1293                fileMap.put(name, schemaDirFile);
1294              }
1295            }
1296
1297            if (! fileMap.isEmpty())
1298            {
1299              return Schema.getSchema(new ArrayList<>(fileMap.values()));
1300            }
1301          }
1302        }
1303      }
1304      catch (final Exception e)
1305      {
1306        Debug.debugException(e);
1307      }
1308    }
1309
1310
1311    // If we've gotten here, then just return null and the tool will try to use
1312    // the default standard schema.
1313    return null;
1314  }
1315
1316
1317
1318  /**
1319   * Creates the entry and change record translators that will be used to
1320   * perform the transformations.
1321   *
1322   * @param  entryTranslators         A list to which all created entry
1323   *                                  translators should be written.
1324   * @param  changeRecordTranslators  A list to which all created change record
1325   *                                  translators should be written.
1326   * @param  schema                   The schema to use when processing.
1327   * @param  excludedEntryCount       A counter used to keep track of the number
1328   *                                  of entries that have been excluded from
1329   *                                  the result set.
1330   */
1331  private void createTranslators(
1332       final List<LDIFReaderEntryTranslator> entryTranslators,
1333       final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators,
1334       final Schema schema, final AtomicLong excludedEntryCount)
1335  {
1336    if (scrambleAttribute.isPresent())
1337    {
1338      final Long seed;
1339      if (randomSeed.isPresent())
1340      {
1341        seed = randomSeed.getValue().longValue();
1342      }
1343      else
1344      {
1345        seed = null;
1346      }
1347
1348      final ScrambleAttributeTransformation t =
1349           new ScrambleAttributeTransformation(schema, seed,
1350                processDNs.isPresent(), scrambleAttribute.getValues(),
1351                scrambleJSONField.getValues());
1352      entryTranslators.add(t);
1353      changeRecordTranslators.add(t);
1354    }
1355
1356    if (sequentialAttribute.isPresent())
1357    {
1358      final long initialValue;
1359      if (initialSequentialValue.isPresent())
1360      {
1361        initialValue = initialSequentialValue.getValue().longValue();
1362      }
1363      else
1364      {
1365        initialValue = 0L;
1366      }
1367
1368      final long incrementAmount;
1369      if (sequentialValueIncrement.isPresent())
1370      {
1371        incrementAmount = sequentialValueIncrement.getValue().longValue();
1372      }
1373      else
1374      {
1375        incrementAmount = 1L;
1376      }
1377
1378      for (final String attrName : sequentialAttribute.getValues())
1379      {
1380
1381
1382        final ReplaceWithCounterTransformation t =
1383             new ReplaceWithCounterTransformation(schema, attrName,
1384                  initialValue, incrementAmount,
1385                  textBeforeSequentialValue.getValue(),
1386                  textAfterSequentialValue.getValue(), processDNs.isPresent());
1387        entryTranslators.add(t);
1388      }
1389    }
1390
1391    if (replaceValuesAttribute.isPresent())
1392    {
1393      final ReplaceAttributeTransformation t =
1394           new ReplaceAttributeTransformation(schema,
1395                replaceValuesAttribute.getValue(),
1396                replacementValue.getValues());
1397      entryTranslators.add(t);
1398    }
1399
1400    if (addAttributeName.isPresent())
1401    {
1402      final AddAttributeTransformation t = new AddAttributeTransformation(
1403           schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(),
1404           addAttributeFilter.getValue(),
1405           new Attribute(addAttributeName.getValue(), schema,
1406                addAttributeValue.getValues()),
1407           (! addToExistingValues.isPresent()));
1408      entryTranslators.add(t);
1409    }
1410
1411    if (renameAttributeFrom.isPresent())
1412    {
1413      final Iterator<String> renameFromIterator =
1414           renameAttributeFrom.getValues().iterator();
1415      final Iterator<String> renameToIterator =
1416           renameAttributeTo.getValues().iterator();
1417      while (renameFromIterator.hasNext())
1418      {
1419        final RenameAttributeTransformation t =
1420             new RenameAttributeTransformation(schema,
1421                  renameFromIterator.next(), renameToIterator.next(),
1422                  processDNs.isPresent());
1423        entryTranslators.add(t);
1424        changeRecordTranslators.add(t);
1425      }
1426    }
1427
1428    if (flattenBaseDN.isPresent())
1429    {
1430      final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation(
1431           schema, flattenBaseDN.getValue(),
1432           flattenAddOmittedRDNAttributesToEntry.isPresent(),
1433           flattenAddOmittedRDNAttributesToRDN.isPresent(),
1434           flattenExcludeFilter.getValue());
1435      entryTranslators.add(t);
1436    }
1437
1438    if (moveSubtreeFrom.isPresent())
1439    {
1440      final Iterator<DN> moveFromIterator =
1441           moveSubtreeFrom.getValues().iterator();
1442      final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator();
1443      while (moveFromIterator.hasNext())
1444      {
1445        final MoveSubtreeTransformation t =
1446             new MoveSubtreeTransformation(moveFromIterator.next(),
1447                  moveToIterator.next());
1448        entryTranslators.add(t);
1449        changeRecordTranslators.add(t);
1450      }
1451    }
1452
1453    if (redactAttribute.isPresent())
1454    {
1455      final RedactAttributeTransformation t = new RedactAttributeTransformation(
1456           schema, processDNs.isPresent(),
1457           (! hideRedactedValueCount.isPresent()), redactAttribute.getValues());
1458      entryTranslators.add(t);
1459      changeRecordTranslators.add(t);
1460    }
1461
1462    if (excludeAttribute.isPresent())
1463    {
1464      final ExcludeAttributeTransformation t =
1465           new ExcludeAttributeTransformation(schema,
1466                excludeAttribute.getValues());
1467      entryTranslators.add(t);
1468      changeRecordTranslators.add(t);
1469    }
1470
1471    if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() ||
1472        excludeEntryFilter.isPresent())
1473    {
1474      final ExcludeEntryTransformation t = new ExcludeEntryTransformation(
1475           schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(),
1476           excludeEntryFilter.getValue(),
1477           (! excludeNonMatchingEntries.isPresent()), excludedEntryCount);
1478      entryTranslators.add(t);
1479    }
1480
1481    if (excludeChangeType.isPresent())
1482    {
1483      final Set<ChangeType> changeTypes = EnumSet.noneOf(ChangeType.class);
1484      for (final String changeTypeName : excludeChangeType.getValues())
1485      {
1486        changeTypes.add(ChangeType.forName(changeTypeName));
1487      }
1488
1489      changeRecordTranslators.add(
1490           new ExcludeChangeTypeTransformation(changeTypes));
1491    }
1492
1493    if (excludeRecordsWithoutChangeType.isPresent())
1494    {
1495      entryTranslators.add(new ExcludeAllEntriesTransformation());
1496    }
1497
1498    entryTranslators.add(this);
1499  }
1500
1501
1502
1503  /**
1504   * {@inheritDoc}
1505   */
1506  @Override()
1507  public LinkedHashMap<String[],String> getExampleUsages()
1508  {
1509    final LinkedHashMap<String[],String> examples =
1510         new LinkedHashMap<>(StaticUtils.computeMapCapacity(4));
1511
1512    examples.put(
1513         new String[]
1514         {
1515           "--sourceLDIF", "input.ldif",
1516           "--targetLDIF", "scrambled.ldif",
1517           "--scrambleAttribute", "givenName",
1518           "--scrambleAttribute", "sn",
1519           "--scrambleAttribute", "cn",
1520           "--numThreads", "10",
1521           "--schemaPath", "/ds/config/schema",
1522           "--processDNs"
1523         },
1524         INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get());
1525
1526    examples.put(
1527         new String[]
1528         {
1529           "--sourceLDIF", "input.ldif",
1530           "--targetLDIF", "sequential.ldif",
1531           "--sequentialAttribute", "uid",
1532           "--initialSequentialValue", "1",
1533           "--sequentialValueIncrement", "1",
1534           "--textBeforeSequentialValue", "user.",
1535           "--numThreads", "10",
1536           "--schemaPath", "/ds/config/schema",
1537           "--processDNs"
1538         },
1539         INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get());
1540
1541    examples.put(
1542         new String[]
1543         {
1544           "--sourceLDIF", "input.ldif",
1545           "--targetLDIF", "added-organization.ldif",
1546           "--addAttributeName", "o",
1547           "--addAttributeValue", "Example Corp.",
1548           "--addAttributeFilter", "(objectClass=person)",
1549           "--numThreads", "10",
1550           "--schemaPath", "/ds/config/schema"
1551         },
1552         INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get());
1553
1554    examples.put(
1555         new String[]
1556         {
1557           "--sourceLDIF", "input.ldif",
1558           "--targetLDIF", "rebased.ldif",
1559           "--moveSubtreeFrom", "o=example.com",
1560           "--moveSubtreeTo", "dc=example,dc=com",
1561           "--numThreads", "10",
1562           "--schemaPath", "/ds/config/schema"
1563         },
1564         INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get());
1565
1566    return examples;
1567  }
1568
1569
1570
1571  /**
1572   * {@inheritDoc}
1573   */
1574  @Override()
1575  public Entry translate(final Entry original, final long firstLineNumber)
1576         throws LDIFException
1577  {
1578    final ByteStringBuffer buffer = getBuffer();
1579    if (wrapColumn.isPresent())
1580    {
1581      original.toLDIF(buffer, wrapColumn.getValue());
1582    }
1583    else
1584    {
1585      original.toLDIF(buffer, 0);
1586    }
1587    buffer.append(StaticUtils.EOL_BYTES);
1588
1589    return new PreEncodedLDIFEntry(original, buffer.toByteArray());
1590  }
1591
1592
1593
1594  /**
1595   * Retrieves a byte string buffer that can be used to perform LDIF encoding.
1596   *
1597   * @return  A byte string buffer that can be used to perform LDIF encoding.
1598   */
1599  private ByteStringBuffer getBuffer()
1600  {
1601    ByteStringBuffer buffer = byteStringBuffers.get();
1602    if (buffer == null)
1603    {
1604      buffer = new ByteStringBuffer();
1605      byteStringBuffers.set(buffer);
1606    }
1607    else
1608    {
1609      buffer.clear();
1610    }
1611
1612    return buffer;
1613  }
1614}