001/* 002 * Copyright 2008-2019 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2019 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.IOException; 026import java.io.OutputStream; 027import java.io.Serializable; 028import java.text.ParseException; 029import java.util.ArrayList; 030import java.util.LinkedHashMap; 031import java.util.List; 032import java.util.Set; 033import java.util.concurrent.CyclicBarrier; 034import java.util.concurrent.atomic.AtomicBoolean; 035import java.util.concurrent.atomic.AtomicInteger; 036import java.util.concurrent.atomic.AtomicLong; 037 038import com.unboundid.ldap.sdk.Control; 039import com.unboundid.ldap.sdk.LDAPConnection; 040import com.unboundid.ldap.sdk.LDAPConnectionOptions; 041import com.unboundid.ldap.sdk.LDAPException; 042import com.unboundid.ldap.sdk.ResultCode; 043import com.unboundid.ldap.sdk.Version; 044import com.unboundid.ldap.sdk.controls.AssertionRequestControl; 045import com.unboundid.ldap.sdk.controls.PermissiveModifyRequestControl; 046import com.unboundid.ldap.sdk.controls.PostReadRequestControl; 047import com.unboundid.ldap.sdk.controls.PreReadRequestControl; 048import com.unboundid.util.ColumnFormatter; 049import com.unboundid.util.Debug; 050import com.unboundid.util.FixedRateBarrier; 051import com.unboundid.util.FormattableColumn; 052import com.unboundid.util.HorizontalAlignment; 053import com.unboundid.util.LDAPCommandLineTool; 054import com.unboundid.util.ObjectPair; 055import com.unboundid.util.OutputFormat; 056import com.unboundid.util.RateAdjustor; 057import com.unboundid.util.ResultCodeCounter; 058import com.unboundid.util.StaticUtils; 059import com.unboundid.util.ThreadSafety; 060import com.unboundid.util.ThreadSafetyLevel; 061import com.unboundid.util.ValuePattern; 062import com.unboundid.util.WakeableSleeper; 063import com.unboundid.util.args.ArgumentException; 064import com.unboundid.util.args.ArgumentParser; 065import com.unboundid.util.args.BooleanArgument; 066import com.unboundid.util.args.ControlArgument; 067import com.unboundid.util.args.FileArgument; 068import com.unboundid.util.args.FilterArgument; 069import com.unboundid.util.args.IntegerArgument; 070import com.unboundid.util.args.StringArgument; 071 072 073 074/** 075 * This class provides a tool that can be used to perform repeated modifications 076 * in an LDAP directory server using multiple threads. It can help provide an 077 * estimate of the modify performance that a directory server is able to 078 * achieve. The target entry DN may be a value pattern as described in the 079 * {@link ValuePattern} class. This makes it possible to modify a range of 080 * entries rather than repeatedly updating the same entry. 081 * <BR><BR> 082 * Some of the APIs demonstrated by this example include: 083 * <UL> 084 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 085 * package)</LI> 086 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 087 * package)</LI> 088 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk} 089 * package)</LI> 090 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI> 091 * </UL> 092 * <BR><BR> 093 * All of the necessary information is provided using command line arguments. 094 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 095 * class, as well as the following additional arguments: 096 * <UL> 097 * <LI>"-b {entryDN}" or "--targetDN {baseDN}" -- specifies the DN of the 098 * entry to be modified. This must be provided. It may be a simple DN, 099 * or it may be a value pattern to express a range of entry DNs.</LI> 100 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of the 101 * attribute to modify. Multiple attributes may be modified by providing 102 * multiple instances of this argument. At least one attribute must be 103 * provided.</LI> 104 * <LI>"--valuePattern {pattern}" -- specifies the pattern to use to generate 105 * the value to use for each modification. If this argument is provided, 106 * then neither the "--valueLength" nor "--characterSet" arguments may be 107 * given.</LI> 108 * <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to 109 * use for the values of the target attributes. If this is not provided, 110 * then a default length of 10 bytes will be used.</LI> 111 * <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of 112 * characters that will be used to generate the values to use for the 113 * target attributes. It should only include ASCII characters. Values 114 * will be generated from randomly-selected characters from this set. If 115 * this is not provided, then a default set of lowercase alphabetic 116 * characters will be used.</LI> 117 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 118 * concurrent threads to use when performing the modifications. If this 119 * is not provided, then a default of one thread will be used.</LI> 120 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of 121 * time in seconds between lines out output. If this is not provided, 122 * then a default interval duration of five seconds will be used.</LI> 123 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of 124 * intervals for which to run. If this is not provided, then it will 125 * run forever.</LI> 126 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of modify 127 * iterations that should be performed on a connection before that 128 * connection is closed and replaced with a newly-established (and 129 * authenticated, if appropriate) connection.</LI> 130 * <LI>"-r {modifies-per-second}" or "--ratePerSecond {modifies-per-second}" 131 * -- specifies the target number of modifies to perform per second. It 132 * is still necessary to specify a sufficient number of threads for 133 * achieving this rate. If this option is not provided, then the tool 134 * will run at the maximum rate for the specified number of threads.</LI> 135 * <LI>"--variableRateData {path}" -- specifies the path to a file containing 136 * information needed to allow the tool to vary the target rate over time. 137 * If this option is not provided, then the tool will either use a fixed 138 * target rate as specified by the "--ratePerSecond" argument, or it will 139 * run at the maximum rate.</LI> 140 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to 141 * which sample data will be written illustrating and describing the 142 * format of the file expected to be used in conjunction with the 143 * "--variableRateData" argument.</LI> 144 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to 145 * complete before beginning overall statistics collection.</LI> 146 * <LI>"--timestampFormat {format}" -- specifies the format to use for 147 * timestamps included before each output line. The format may be one of 148 * "none" (for no timestamps), "with-date" (to include both the date and 149 * the time), or "without-date" (to include only time time).</LI> 150 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied 151 * authorization v2 control to request that the operation be processed 152 * using an alternate authorization identity. In this case, the bind DN 153 * should be that of a user that has permission to use this control. The 154 * authorization identity may be a value pattern.</LI> 155 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the 156 * result codes for failed operations should not be displayed.</LI> 157 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a 158 * display-friendly format.</LI> 159 * </UL> 160 */ 161@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 162public final class ModRate 163 extends LDAPCommandLineTool 164 implements Serializable 165{ 166 /** 167 * The serial version UID for this serializable class. 168 */ 169 private static final long serialVersionUID = 2709717414202815822L; 170 171 172 173 // Indicates whether a request has been made to stop running. 174 private final AtomicBoolean stopRequested; 175 176 // The number of modrate threads that are currently running. 177 private final AtomicInteger runningThreads; 178 179 // The argument used to indicate whether to generate output in CSV format. 180 private BooleanArgument csvFormat; 181 182 // Indicates that the tool should use the increment modification type instead 183 // of replace. 184 private BooleanArgument increment; 185 186 // Indicates that modify requests should include the permissive modify request 187 // control. 188 private BooleanArgument permissiveModify; 189 190 // The argument used to indicate whether to suppress information about error 191 // result codes. 192 private BooleanArgument suppressErrorsArgument; 193 194 // The argument used to indicate that a generic control should be included in 195 // the request. 196 private ControlArgument control; 197 198 // The argument used to specify a variable rate file. 199 private FileArgument sampleRateFile; 200 201 // The argument used to specify a variable rate file. 202 private FileArgument variableRateData; 203 204 // Indicates that modify requests should include the assertion request control 205 // with the specified filter. 206 private FilterArgument assertionFilter; 207 208 // The argument used to specify the collection interval. 209 private IntegerArgument collectionInterval; 210 211 // The increment amount to use when performing an increment instead of a 212 // replace. 213 private IntegerArgument incrementAmount; 214 215 // The argument used to specify the number of modify iterations on a 216 // connection before it is closed and re-established. 217 private IntegerArgument iterationsBeforeReconnect; 218 219 // The argument used to specify the number of intervals. 220 private IntegerArgument numIntervals; 221 222 // The argument used to specify the number of threads. 223 private IntegerArgument numThreads; 224 225 // The argument used to specify the seed to use for the random number 226 // generator. 227 private IntegerArgument randomSeed; 228 229 // The target rate of modifies per second. 230 private IntegerArgument ratePerSecond; 231 232 // The number of values to include in the replace modification. 233 private IntegerArgument valueCount; 234 235 // The argument used to specify the length of the values to generate. 236 private IntegerArgument valueLength; 237 238 // The number of warm-up intervals to perform. 239 private IntegerArgument warmUpIntervals; 240 241 // The argument used to specify the name of the attribute to modify. 242 private StringArgument attribute; 243 244 // The argument used to specify the set of characters to use when generating 245 // values. 246 private StringArgument characterSet; 247 248 // The argument used to specify the DNs of the entries to modify. 249 private StringArgument entryDN; 250 251 // Indicates that modify requests should include the post-read request control 252 // to request the specified attribute. 253 private StringArgument postReadAttribute; 254 255 // Indicates that modify requests should include the pre-read request control 256 // to request the specified attribute. 257 private StringArgument preReadAttribute; 258 259 // The argument used to specify the proxied authorization identity. 260 private StringArgument proxyAs; 261 262 // The argument used to specify the timestamp format. 263 private StringArgument timestampFormat; 264 265 // The argument used to specify the pattern to use to generate values. 266 private StringArgument valuePattern; 267 268 // A wakeable sleeper that will be used to sleep between reporting intervals. 269 private final WakeableSleeper sleeper; 270 271 272 273 /** 274 * Parse the provided command line arguments and make the appropriate set of 275 * changes. 276 * 277 * @param args The command line arguments provided to this program. 278 */ 279 public static void main(final String[] args) 280 { 281 final ResultCode resultCode = main(args, System.out, System.err); 282 if (resultCode != ResultCode.SUCCESS) 283 { 284 System.exit(resultCode.intValue()); 285 } 286 } 287 288 289 290 /** 291 * Parse the provided command line arguments and make the appropriate set of 292 * changes. 293 * 294 * @param args The command line arguments provided to this program. 295 * @param outStream The output stream to which standard out should be 296 * written. It may be {@code null} if output should be 297 * suppressed. 298 * @param errStream The output stream to which standard error should be 299 * written. It may be {@code null} if error messages 300 * should be suppressed. 301 * 302 * @return A result code indicating whether the processing was successful. 303 */ 304 public static ResultCode main(final String[] args, 305 final OutputStream outStream, 306 final OutputStream errStream) 307 { 308 final ModRate modRate = new ModRate(outStream, errStream); 309 return modRate.runTool(args); 310 } 311 312 313 314 /** 315 * Creates a new instance of this tool. 316 * 317 * @param outStream The output stream to which standard out should be 318 * written. It may be {@code null} if output should be 319 * suppressed. 320 * @param errStream The output stream to which standard error should be 321 * written. It may be {@code null} if error messages 322 * should be suppressed. 323 */ 324 public ModRate(final OutputStream outStream, final OutputStream errStream) 325 { 326 super(outStream, errStream); 327 328 stopRequested = new AtomicBoolean(false); 329 runningThreads = new AtomicInteger(0); 330 sleeper = new WakeableSleeper(); 331 } 332 333 334 335 /** 336 * Retrieves the name for this tool. 337 * 338 * @return The name for this tool. 339 */ 340 @Override() 341 public String getToolName() 342 { 343 return "modrate"; 344 } 345 346 347 348 /** 349 * Retrieves the description for this tool. 350 * 351 * @return The description for this tool. 352 */ 353 @Override() 354 public String getToolDescription() 355 { 356 return "Perform repeated modifications against " + 357 "an LDAP directory server."; 358 } 359 360 361 362 /** 363 * Retrieves the version string for this tool. 364 * 365 * @return The version string for this tool. 366 */ 367 @Override() 368 public String getToolVersion() 369 { 370 return Version.NUMERIC_VERSION_STRING; 371 } 372 373 374 375 /** 376 * Indicates whether this tool should provide support for an interactive mode, 377 * in which the tool offers a mode in which the arguments can be provided in 378 * a text-driven menu rather than requiring them to be given on the command 379 * line. If interactive mode is supported, it may be invoked using the 380 * "--interactive" argument. Alternately, if interactive mode is supported 381 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 382 * interactive mode may be invoked by simply launching the tool without any 383 * arguments. 384 * 385 * @return {@code true} if this tool supports interactive mode, or 386 * {@code false} if not. 387 */ 388 @Override() 389 public boolean supportsInteractiveMode() 390 { 391 return true; 392 } 393 394 395 396 /** 397 * Indicates whether this tool defaults to launching in interactive mode if 398 * the tool is invoked without any command-line arguments. This will only be 399 * used if {@link #supportsInteractiveMode()} returns {@code true}. 400 * 401 * @return {@code true} if this tool defaults to using interactive mode if 402 * launched without any command-line arguments, or {@code false} if 403 * not. 404 */ 405 @Override() 406 public boolean defaultsToInteractiveMode() 407 { 408 return true; 409 } 410 411 412 413 /** 414 * Indicates whether this tool should provide arguments for redirecting output 415 * to a file. If this method returns {@code true}, then the tool will offer 416 * an "--outputFile" argument that will specify the path to a file to which 417 * all standard output and standard error content will be written, and it will 418 * also offer a "--teeToStandardOut" argument that can only be used if the 419 * "--outputFile" argument is present and will cause all output to be written 420 * to both the specified output file and to standard output. 421 * 422 * @return {@code true} if this tool should provide arguments for redirecting 423 * output to a file, or {@code false} if not. 424 */ 425 @Override() 426 protected boolean supportsOutputFile() 427 { 428 return true; 429 } 430 431 432 433 /** 434 * Indicates whether this tool should default to interactively prompting for 435 * the bind password if a password is required but no argument was provided 436 * to indicate how to get the password. 437 * 438 * @return {@code true} if this tool should default to interactively 439 * prompting for the bind password, or {@code false} if not. 440 */ 441 @Override() 442 protected boolean defaultToPromptForBindPassword() 443 { 444 return true; 445 } 446 447 448 449 /** 450 * Indicates whether this tool supports the use of a properties file for 451 * specifying default values for arguments that aren't specified on the 452 * command line. 453 * 454 * @return {@code true} if this tool supports the use of a properties file 455 * for specifying default values for arguments that aren't specified 456 * on the command line, or {@code false} if not. 457 */ 458 @Override() 459 public boolean supportsPropertiesFile() 460 { 461 return true; 462 } 463 464 465 466 /** 467 * Indicates whether the LDAP-specific arguments should include alternate 468 * versions of all long identifiers that consist of multiple words so that 469 * they are available in both camelCase and dash-separated versions. 470 * 471 * @return {@code true} if this tool should provide multiple versions of 472 * long identifiers for LDAP-specific arguments, or {@code false} if 473 * not. 474 */ 475 @Override() 476 protected boolean includeAlternateLongIdentifiers() 477 { 478 return true; 479 } 480 481 482 483 /** 484 * {@inheritDoc} 485 */ 486 @Override() 487 protected boolean logToolInvocationByDefault() 488 { 489 return true; 490 } 491 492 493 494 /** 495 * Adds the arguments used by this program that aren't already provided by the 496 * generic {@code LDAPCommandLineTool} framework. 497 * 498 * @param parser The argument parser to which the arguments should be added. 499 * 500 * @throws ArgumentException If a problem occurs while adding the arguments. 501 */ 502 @Override() 503 public void addNonLDAPArguments(final ArgumentParser parser) 504 throws ArgumentException 505 { 506 String description = "The DN of the entry to modify. It may be a simple " + 507 "DN or a value pattern to specify a range of DN (e.g., " + 508 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " + 509 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " + 510 "value pattern syntax. This must be provided."; 511 entryDN = new StringArgument('b', "entryDN", true, 1, "{dn}", description); 512 entryDN.setArgumentGroupName("Modification Arguments"); 513 entryDN.addLongIdentifier("entry-dn", true); 514 parser.addArgument(entryDN); 515 516 517 description = "The name of the attribute to modify. Multiple attributes " + 518 "may be specified by providing this argument multiple " + 519 "times. At least one attribute must be specified."; 520 attribute = new StringArgument('A', "attribute", true, 0, "{name}", 521 description); 522 attribute.setArgumentGroupName("Modification Arguments"); 523 parser.addArgument(attribute); 524 525 526 description = "The pattern to use to generate values for the replace " + 527 "modifications. If this is provided, then neither the " + 528 "--valueLength argument nor the --characterSet arguments " + 529 "may be provided."; 530 valuePattern = new StringArgument(null, "valuePattern", false, 1, 531 "{pattern}", description); 532 valuePattern.setArgumentGroupName("Modification Arguments"); 533 valuePattern.addLongIdentifier("value-pattern", true); 534 parser.addArgument(valuePattern); 535 536 537 description = "The length in bytes to use when generating values for the " + 538 "replace modifications. If this is not provided, then a " + 539 "default length of ten bytes will be used."; 540 valueLength = new IntegerArgument('l', "valueLength", false, 1, "{num}", 541 description, 1, Integer.MAX_VALUE); 542 valueLength.setArgumentGroupName("Modification Arguments"); 543 valueLength.addLongIdentifier("value-length", true); 544 parser.addArgument(valueLength); 545 546 547 description = "The number of values to include in replace " + 548 "modifications. If this is not provided, then a default " + 549 "of one value will be used."; 550 valueCount = new IntegerArgument(null, "valueCount", false, 1, "{num}", 551 description, 0, Integer.MAX_VALUE, 1); 552 valueCount.setArgumentGroupName("Modification Arguments"); 553 valueCount.addLongIdentifier("value-count", true); 554 parser.addArgument(valueCount); 555 556 557 description = "Indicates that the tool should use the increment " + 558 "modification type rather than the replace modification " + 559 "type."; 560 increment = new BooleanArgument(null, "increment", 1, description); 561 increment.setArgumentGroupName("Modification Arguments"); 562 parser.addArgument(increment); 563 564 565 description = "The amount by which to increment values when using the " + 566 "increment modification type. The amount may be negative " + 567 "if values should be decremented rather than incremented. " + 568 "If this is not provided, then a default increment amount " + 569 "of one will be used."; 570 incrementAmount = new IntegerArgument(null, "incrementAmount", false, 1, 571 null, description, Integer.MIN_VALUE, 572 Integer.MAX_VALUE, 1); 573 incrementAmount.setArgumentGroupName("Modification Arguments"); 574 incrementAmount.addLongIdentifier("increment-amount", true); 575 parser.addArgument(incrementAmount); 576 577 578 description = "The set of characters to use to generate the values for " + 579 "the modifications. It should only include ASCII " + 580 "characters. If this is not provided, then a default set " + 581 "of lowercase alphabetic characters will be used."; 582 characterSet = new StringArgument('C', "characterSet", false, 1, "{chars}", 583 description); 584 characterSet.setArgumentGroupName("Modification Arguments"); 585 characterSet.addLongIdentifier("character-set", true); 586 parser.addArgument(characterSet); 587 588 589 description = "Indicates that modify requests should include the " + 590 "assertion request control with the specified filter."; 591 assertionFilter = new FilterArgument(null, "assertionFilter", false, 1, 592 "{filter}", description); 593 assertionFilter.setArgumentGroupName("Request Control Arguments"); 594 assertionFilter.addLongIdentifier("assertion-filter", true); 595 parser.addArgument(assertionFilter); 596 597 598 description = "Indicates that modify requests should include the " + 599 "permissive modify request control."; 600 permissiveModify = new BooleanArgument(null, "permissiveModify", 1, 601 description); 602 permissiveModify.setArgumentGroupName("Request Control Arguments"); 603 permissiveModify.addLongIdentifier("permissive-modify", true); 604 parser.addArgument(permissiveModify); 605 606 607 description = "Indicates that modify requests should include the " + 608 "pre-read request control with the specified requested " + 609 "attribute. This argument may be provided multiple times " + 610 "to indicate that multiple requested attributes should be " + 611 "included in the pre-read request control."; 612 preReadAttribute = new StringArgument(null, "preReadAttribute", false, 0, 613 "{attribute}", description); 614 preReadAttribute.setArgumentGroupName("Request Control Arguments"); 615 preReadAttribute.addLongIdentifier("pre-read-attribute", true); 616 parser.addArgument(preReadAttribute); 617 618 619 description = "Indicates that modify requests should include the " + 620 "post-read request control with the specified requested " + 621 "attribute. This argument may be provided multiple times " + 622 "to indicate that multiple requested attributes should be " + 623 "included in the post-read request control."; 624 postReadAttribute = new StringArgument(null, "postReadAttribute", false, 0, 625 "{attribute}", description); 626 postReadAttribute.setArgumentGroupName("Request Control Arguments"); 627 postReadAttribute.addLongIdentifier("post-read-attribute", true); 628 parser.addArgument(postReadAttribute); 629 630 631 description = "Indicates that the proxied authorization control (as " + 632 "defined in RFC 4370) should be used to request that " + 633 "operations be processed using an alternate authorization " + 634 "identity. This may be a simple authorization ID or it " + 635 "may be a value pattern to specify a range of " + 636 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL + 637 " for complete details about the value pattern syntax."; 638 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}", 639 description); 640 proxyAs.setArgumentGroupName("Request Control Arguments"); 641 proxyAs.addLongIdentifier("proxy-as", true); 642 parser.addArgument(proxyAs); 643 644 645 description = "Indicates that modify requests should include the " + 646 "specified request control. This may be provided multiple " + 647 "times to include multiple request controls."; 648 control = new ControlArgument('J', "control", false, 0, null, description); 649 control.setArgumentGroupName("Request Control Arguments"); 650 parser.addArgument(control); 651 652 653 description = "The number of threads to use to perform the " + 654 "modifications. If this is not provided, a single thread " + 655 "will be used."; 656 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 657 description, 1, Integer.MAX_VALUE, 1); 658 numThreads.setArgumentGroupName("Rate Management Arguments"); 659 numThreads.addLongIdentifier("num-threads", true); 660 parser.addArgument(numThreads); 661 662 663 description = "The length of time in seconds between output lines. If " + 664 "this is not provided, then a default interval of five " + 665 "seconds will be used."; 666 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1, 667 "{num}", description, 1, 668 Integer.MAX_VALUE, 5); 669 collectionInterval.setArgumentGroupName("Rate Management Arguments"); 670 collectionInterval.addLongIdentifier("interval-duration", true); 671 parser.addArgument(collectionInterval); 672 673 674 description = "The maximum number of intervals for which to run. If " + 675 "this is not provided, then the tool will run until it is " + 676 "interrupted."; 677 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}", 678 description, 1, Integer.MAX_VALUE, 679 Integer.MAX_VALUE); 680 numIntervals.setArgumentGroupName("Rate Management Arguments"); 681 numIntervals.addLongIdentifier("num-intervals", true); 682 parser.addArgument(numIntervals); 683 684 description = "The number of modify iterations that should be processed " + 685 "on a connection before that connection is closed and " + 686 "replaced with a newly-established (and authenticated, if " + 687 "appropriate) connection. If this is not provided, then " + 688 "connections will not be periodically closed and " + 689 "re-established."; 690 iterationsBeforeReconnect = new IntegerArgument(null, 691 "iterationsBeforeReconnect", false, 1, "{num}", description, 0); 692 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments"); 693 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect", 694 true); 695 parser.addArgument(iterationsBeforeReconnect); 696 697 description = "The target number of modifies to perform per second. It " + 698 "is still necessary to specify a sufficient number of " + 699 "threads for achieving this rate. If neither this option " + 700 "nor --variableRateData is provided, then the tool will " + 701 "run at the maximum rate for the specified number of " + 702 "threads."; 703 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1, 704 "{modifies-per-second}", description, 705 1, Integer.MAX_VALUE); 706 ratePerSecond.setArgumentGroupName("Rate Management Arguments"); 707 ratePerSecond.addLongIdentifier("rate-per-second", true); 708 parser.addArgument(ratePerSecond); 709 710 final String variableRateDataArgName = "variableRateData"; 711 final String generateSampleRateFileArgName = "generateSampleRateFile"; 712 description = RateAdjustor.getVariableRateDataArgumentDescription( 713 generateSampleRateFileArgName); 714 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1, 715 "{path}", description, true, true, true, 716 false); 717 variableRateData.setArgumentGroupName("Rate Management Arguments"); 718 variableRateData.addLongIdentifier("variable-rate-data", true); 719 parser.addArgument(variableRateData); 720 721 description = RateAdjustor.getGenerateSampleVariableRateFileDescription( 722 variableRateDataArgName); 723 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName, 724 false, 1, "{path}", description, false, 725 true, true, false); 726 sampleRateFile.setArgumentGroupName("Rate Management Arguments"); 727 sampleRateFile.addLongIdentifier("generate-sample-rate-file", true); 728 sampleRateFile.setUsageArgument(true); 729 parser.addArgument(sampleRateFile); 730 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile); 731 732 description = "The number of intervals to complete before beginning " + 733 "overall statistics collection. Specifying a nonzero " + 734 "number of warm-up intervals gives the client and server " + 735 "a chance to warm up without skewing performance results."; 736 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1, 737 "{num}", description, 0, Integer.MAX_VALUE, 0); 738 warmUpIntervals.setArgumentGroupName("Rate Management Arguments"); 739 warmUpIntervals.addLongIdentifier("warm-up-intervals", true); 740 parser.addArgument(warmUpIntervals); 741 742 description = "Indicates the format to use for timestamps included in " + 743 "the output. A value of 'none' indicates that no " + 744 "timestamps should be included. A value of 'with-date' " + 745 "indicates that both the date and the time should be " + 746 "included. A value of 'without-date' indicates that only " + 747 "the time should be included."; 748 final Set<String> allowedFormats = 749 StaticUtils.setOf("none", "with-date", "without-date"); 750 timestampFormat = new StringArgument(null, "timestampFormat", true, 1, 751 "{format}", description, allowedFormats, "none"); 752 timestampFormat.addLongIdentifier("timestamp-format", true); 753 parser.addArgument(timestampFormat); 754 755 description = "Indicates that information about the result codes for " + 756 "failed operations should not be displayed."; 757 suppressErrorsArgument = new BooleanArgument(null, 758 "suppressErrorResultCodes", 1, description); 759 suppressErrorsArgument.addLongIdentifier("suppress-error-result-codes", 760 true); 761 parser.addArgument(suppressErrorsArgument); 762 763 description = "Generate output in CSV format rather than a " + 764 "display-friendly format"; 765 csvFormat = new BooleanArgument('c', "csv", 1, description); 766 parser.addArgument(csvFormat); 767 768 description = "Specifies the seed to use for the random number generator."; 769 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}", 770 description); 771 randomSeed.addLongIdentifier("random-seed", true); 772 parser.addArgument(randomSeed); 773 774 775 // The incrementAmount argument can only be used if the increment argument 776 // is provided. 777 parser.addDependentArgumentSet(incrementAmount, increment); 778 779 780 // None of the valueLength, valueCount, characterSet, or valuePattern 781 // arguments can be used if the increment argument is provided. 782 parser.addExclusiveArgumentSet(increment, valueLength); 783 parser.addExclusiveArgumentSet(increment, valueCount); 784 parser.addExclusiveArgumentSet(increment, characterSet); 785 parser.addExclusiveArgumentSet(increment, valuePattern); 786 787 788 // The valuePattern argument cannot be used with either the valueLength or 789 // characterSet arguments. 790 parser.addExclusiveArgumentSet(valuePattern, valueLength); 791 parser.addExclusiveArgumentSet(valuePattern, characterSet); 792 } 793 794 795 796 /** 797 * Indicates whether this tool supports creating connections to multiple 798 * servers. If it is to support multiple servers, then the "--hostname" and 799 * "--port" arguments will be allowed to be provided multiple times, and 800 * will be required to be provided the same number of times. The same type of 801 * communication security and bind credentials will be used for all servers. 802 * 803 * @return {@code true} if this tool supports creating connections to 804 * multiple servers, or {@code false} if not. 805 */ 806 @Override() 807 protected boolean supportsMultipleServers() 808 { 809 return true; 810 } 811 812 813 814 /** 815 * Retrieves the connection options that should be used for connections 816 * created for use with this tool. 817 * 818 * @return The connection options that should be used for connections created 819 * for use with this tool. 820 */ 821 @Override() 822 public LDAPConnectionOptions getConnectionOptions() 823 { 824 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 825 options.setUseSynchronousMode(true); 826 return options; 827 } 828 829 830 831 /** 832 * Performs the actual processing for this tool. In this case, it gets a 833 * connection to the directory server and uses it to perform the requested 834 * modifications. 835 * 836 * @return The result code for the processing that was performed. 837 */ 838 @Override() 839 public ResultCode doToolProcessing() 840 { 841 // If the sample rate file argument was specified, then generate the sample 842 // variable rate data file and return. 843 if (sampleRateFile.isPresent()) 844 { 845 try 846 { 847 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue()); 848 return ResultCode.SUCCESS; 849 } 850 catch (final Exception e) 851 { 852 Debug.debugException(e); 853 err("An error occurred while trying to write sample variable data " + 854 "rate file '", sampleRateFile.getValue().getAbsolutePath(), 855 "': ", StaticUtils.getExceptionMessage(e)); 856 return ResultCode.LOCAL_ERROR; 857 } 858 } 859 860 861 // Determine the random seed to use. 862 final Long seed; 863 if (randomSeed.isPresent()) 864 { 865 seed = Long.valueOf(randomSeed.getValue()); 866 } 867 else 868 { 869 seed = null; 870 } 871 872 // Create the value patterns for the target entry DN and proxied 873 // authorization identities. 874 final ValuePattern dnPattern; 875 try 876 { 877 dnPattern = new ValuePattern(entryDN.getValue(), seed); 878 } 879 catch (final ParseException pe) 880 { 881 Debug.debugException(pe); 882 err("Unable to parse the entry DN value pattern: ", pe.getMessage()); 883 return ResultCode.PARAM_ERROR; 884 } 885 886 final ValuePattern authzIDPattern; 887 if (proxyAs.isPresent()) 888 { 889 try 890 { 891 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed); 892 } 893 catch (final ParseException pe) 894 { 895 Debug.debugException(pe); 896 err("Unable to parse the proxied authorization pattern: ", 897 pe.getMessage()); 898 return ResultCode.PARAM_ERROR; 899 } 900 } 901 else 902 { 903 authzIDPattern = null; 904 } 905 906 907 // Get the set of controls to include in modify requests. 908 final ArrayList<Control> controlList = new ArrayList<>(5); 909 if (assertionFilter.isPresent()) 910 { 911 controlList.add(new AssertionRequestControl(assertionFilter.getValue())); 912 } 913 914 if (permissiveModify.isPresent()) 915 { 916 controlList.add(new PermissiveModifyRequestControl()); 917 } 918 919 if (preReadAttribute.isPresent()) 920 { 921 final List<String> attrList = preReadAttribute.getValues(); 922 final String[] attrArray = new String[attrList.size()]; 923 attrList.toArray(attrArray); 924 controlList.add(new PreReadRequestControl(attrArray)); 925 } 926 927 if (postReadAttribute.isPresent()) 928 { 929 final List<String> attrList = postReadAttribute.getValues(); 930 final String[] attrArray = new String[attrList.size()]; 931 attrList.toArray(attrArray); 932 controlList.add(new PostReadRequestControl(attrArray)); 933 } 934 935 if (control.isPresent()) 936 { 937 controlList.addAll(control.getValues()); 938 } 939 940 final Control[] controlArray = new Control[controlList.size()]; 941 controlList.toArray(controlArray); 942 943 944 // Get the names of the attributes to modify. 945 final String[] attrs = new String[attribute.getValues().size()]; 946 attribute.getValues().toArray(attrs); 947 948 949 // If the --ratePerSecond option was specified, then limit the rate 950 // accordingly. 951 FixedRateBarrier fixedRateBarrier = null; 952 if (ratePerSecond.isPresent() || variableRateData.isPresent()) 953 { 954 // We might not have a rate per second if --variableRateData is specified. 955 // The rate typically doesn't matter except when we have warm-up 956 // intervals. In this case, we'll run at the max rate. 957 final int intervalSeconds = collectionInterval.getValue(); 958 final int ratePerInterval = 959 (ratePerSecond.getValue() == null) 960 ? Integer.MAX_VALUE 961 : ratePerSecond.getValue() * intervalSeconds; 962 fixedRateBarrier = 963 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval); 964 } 965 966 967 // If --variableRateData was specified, then initialize a RateAdjustor. 968 RateAdjustor rateAdjustor = null; 969 if (variableRateData.isPresent()) 970 { 971 try 972 { 973 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier, 974 ratePerSecond.getValue(), variableRateData.getValue()); 975 } 976 catch (final IOException | IllegalArgumentException e) 977 { 978 Debug.debugException(e); 979 err("Initializing the variable rates failed: " + e.getMessage()); 980 return ResultCode.PARAM_ERROR; 981 } 982 } 983 984 985 // Determine whether to include timestamps in the output and if so what 986 // format should be used for them. 987 final boolean includeTimestamp; 988 final String timeFormat; 989 if (timestampFormat.getValue().equalsIgnoreCase("with-date")) 990 { 991 includeTimestamp = true; 992 timeFormat = "dd/MM/yyyy HH:mm:ss"; 993 } 994 else if (timestampFormat.getValue().equalsIgnoreCase("without-date")) 995 { 996 includeTimestamp = true; 997 timeFormat = "HH:mm:ss"; 998 } 999 else 1000 { 1001 includeTimestamp = false; 1002 timeFormat = null; 1003 } 1004 1005 1006 // Determine whether any warm-up intervals should be run. 1007 final long totalIntervals; 1008 final boolean warmUp; 1009 int remainingWarmUpIntervals = warmUpIntervals.getValue(); 1010 if (remainingWarmUpIntervals > 0) 1011 { 1012 warmUp = true; 1013 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals; 1014 } 1015 else 1016 { 1017 warmUp = true; 1018 totalIntervals = 0L + numIntervals.getValue(); 1019 } 1020 1021 1022 // Create the table that will be used to format the output. 1023 final OutputFormat outputFormat; 1024 if (csvFormat.isPresent()) 1025 { 1026 outputFormat = OutputFormat.CSV; 1027 } 1028 else 1029 { 1030 outputFormat = OutputFormat.COLUMNS; 1031 } 1032 1033 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp, 1034 timeFormat, outputFormat, " ", 1035 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1036 "Mods/Sec"), 1037 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1038 "Avg Dur ms"), 1039 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1040 "Errors/Sec"), 1041 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1042 "Mods/Sec"), 1043 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1044 "Avg Dur ms")); 1045 1046 1047 // Create values to use for statistics collection. 1048 final AtomicLong modCounter = new AtomicLong(0L); 1049 final AtomicLong errorCounter = new AtomicLong(0L); 1050 final AtomicLong modDurations = new AtomicLong(0L); 1051 final ResultCodeCounter rcCounter = new ResultCodeCounter(); 1052 1053 1054 // Determine the length of each interval in milliseconds. 1055 final long intervalMillis = 1000L * collectionInterval.getValue(); 1056 1057 1058 // Create the threads to use for the modifications. 1059 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1); 1060 final ModRateThread[] threads = new ModRateThread[numThreads.getValue()]; 1061 for (int i=0; i < threads.length; i++) 1062 { 1063 final LDAPConnection connection; 1064 try 1065 { 1066 connection = getConnection(); 1067 } 1068 catch (final LDAPException le) 1069 { 1070 Debug.debugException(le); 1071 err("Unable to connect to the directory server: ", 1072 StaticUtils.getExceptionMessage(le)); 1073 return le.getResultCode(); 1074 } 1075 1076 final String valuePatternString; 1077 if (valuePattern.isPresent()) 1078 { 1079 valuePatternString = valuePattern.getValue(); 1080 } 1081 else 1082 { 1083 final int length; 1084 if (valueLength.isPresent()) 1085 { 1086 length = valueLength.getValue(); 1087 } 1088 else 1089 { 1090 length = 10; 1091 } 1092 1093 final String charSet; 1094 if (characterSet.isPresent()) 1095 { 1096 charSet = 1097 characterSet.getValue().replace("]", "]]").replace("[", "[["); 1098 } 1099 else 1100 { 1101 charSet = "abcdefghijklmnopqrstuvwxyz"; 1102 } 1103 1104 valuePatternString = "[random:" + length + ':' + charSet + ']'; 1105 } 1106 1107 final ValuePattern parsedValuePattern; 1108 try 1109 { 1110 parsedValuePattern = new ValuePattern(valuePatternString); 1111 } 1112 catch (final ParseException e) 1113 { 1114 Debug.debugException(e); 1115 err(e.getMessage()); 1116 return ResultCode.PARAM_ERROR; 1117 } 1118 1119 threads[i] = new ModRateThread(this, i, connection, dnPattern, attrs, 1120 parsedValuePattern, valueCount.getValue(), increment.isPresent(), 1121 incrementAmount.getValue(), controlArray, authzIDPattern, 1122 iterationsBeforeReconnect.getValue(), runningThreads, barrier, 1123 modCounter, modDurations, errorCounter, rcCounter, fixedRateBarrier); 1124 threads[i].start(); 1125 } 1126 1127 1128 // Display the table header. 1129 for (final String headerLine : formatter.getHeaderLines(true)) 1130 { 1131 out(headerLine); 1132 } 1133 1134 1135 // Start the RateAdjustor before the threads so that the initial value is 1136 // in place before any load is generated unless we're doing a warm-up in 1137 // which case, we'll start it after the warm-up is complete. 1138 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0)) 1139 { 1140 rateAdjustor.start(); 1141 } 1142 1143 1144 // Indicate that the threads can start running. 1145 try 1146 { 1147 barrier.await(); 1148 } 1149 catch (final Exception e) 1150 { 1151 Debug.debugException(e); 1152 } 1153 1154 long overallStartTime = System.nanoTime(); 1155 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis; 1156 1157 1158 boolean setOverallStartTime = false; 1159 long lastDuration = 0L; 1160 long lastNumErrors = 0L; 1161 long lastNumMods = 0L; 1162 long lastEndTime = System.nanoTime(); 1163 for (long i=0; i < totalIntervals; i++) 1164 { 1165 if (rateAdjustor != null) 1166 { 1167 if (! rateAdjustor.isAlive()) 1168 { 1169 out("All of the rates in " + variableRateData.getValue().getName() + 1170 " have been completed."); 1171 break; 1172 } 1173 } 1174 1175 final long startTimeMillis = System.currentTimeMillis(); 1176 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis; 1177 nextIntervalStartTime += intervalMillis; 1178 if (sleepTimeMillis > 0) 1179 { 1180 sleeper.sleep(sleepTimeMillis); 1181 } 1182 1183 if (stopRequested.get()) 1184 { 1185 break; 1186 } 1187 1188 final long endTime = System.nanoTime(); 1189 final long intervalDuration = endTime - lastEndTime; 1190 1191 final long numMods; 1192 final long numErrors; 1193 final long totalDuration; 1194 if (warmUp && (remainingWarmUpIntervals > 0)) 1195 { 1196 numMods = modCounter.getAndSet(0L); 1197 numErrors = errorCounter.getAndSet(0L); 1198 totalDuration = modDurations.getAndSet(0L); 1199 } 1200 else 1201 { 1202 numMods = modCounter.get(); 1203 numErrors = errorCounter.get(); 1204 totalDuration = modDurations.get(); 1205 } 1206 1207 final long recentNumMods = numMods - lastNumMods; 1208 final long recentNumErrors = numErrors - lastNumErrors; 1209 final long recentDuration = totalDuration - lastDuration; 1210 1211 final double numSeconds = intervalDuration / 1_000_000_000.0d; 1212 final double recentModRate = recentNumMods / numSeconds; 1213 final double recentErrorRate = recentNumErrors / numSeconds; 1214 1215 final double recentAvgDuration; 1216 if (recentNumMods > 0L) 1217 { 1218 recentAvgDuration = 1.0d * recentDuration / recentNumMods / 1_000_000; 1219 } 1220 else 1221 { 1222 recentAvgDuration = 0.0d; 1223 } 1224 1225 if (warmUp && (remainingWarmUpIntervals > 0)) 1226 { 1227 out(formatter.formatRow(recentModRate, recentAvgDuration, 1228 recentErrorRate, "warming up", "warming up")); 1229 1230 remainingWarmUpIntervals--; 1231 if (remainingWarmUpIntervals == 0) 1232 { 1233 out("Warm-up completed. Beginning overall statistics collection."); 1234 setOverallStartTime = true; 1235 if (rateAdjustor != null) 1236 { 1237 rateAdjustor.start(); 1238 } 1239 } 1240 } 1241 else 1242 { 1243 if (setOverallStartTime) 1244 { 1245 overallStartTime = lastEndTime; 1246 setOverallStartTime = false; 1247 } 1248 1249 final double numOverallSeconds = 1250 (endTime - overallStartTime) / 1_000_000_000.0d; 1251 final double overallAuthRate = numMods / numOverallSeconds; 1252 1253 final double overallAvgDuration; 1254 if (numMods > 0L) 1255 { 1256 overallAvgDuration = 1.0d * totalDuration / numMods / 1_000_000; 1257 } 1258 else 1259 { 1260 overallAvgDuration = 0.0d; 1261 } 1262 1263 out(formatter.formatRow(recentModRate, recentAvgDuration, 1264 recentErrorRate, overallAuthRate, overallAvgDuration)); 1265 1266 lastNumMods = numMods; 1267 lastNumErrors = numErrors; 1268 lastDuration = totalDuration; 1269 } 1270 1271 final List<ObjectPair<ResultCode,Long>> rcCounts = 1272 rcCounter.getCounts(true); 1273 if ((! suppressErrorsArgument.isPresent()) && (! rcCounts.isEmpty())) 1274 { 1275 err("\tError Results:"); 1276 for (final ObjectPair<ResultCode,Long> p : rcCounts) 1277 { 1278 err("\t", p.getFirst().getName(), ": ", p.getSecond()); 1279 } 1280 } 1281 1282 lastEndTime = endTime; 1283 } 1284 1285 // Shut down the RateAdjustor if we have one. 1286 if (rateAdjustor != null) 1287 { 1288 rateAdjustor.shutDown(); 1289 } 1290 1291 // Stop all of the threads. 1292 ResultCode resultCode = ResultCode.SUCCESS; 1293 for (final ModRateThread t : threads) 1294 { 1295 final ResultCode r = t.stopRunning(); 1296 if (resultCode == ResultCode.SUCCESS) 1297 { 1298 resultCode = r; 1299 } 1300 } 1301 1302 return resultCode; 1303 } 1304 1305 1306 1307 /** 1308 * Requests that this tool stop running. This method will attempt to wait 1309 * for all threads to complete before returning control to the caller. 1310 */ 1311 public void stopRunning() 1312 { 1313 stopRequested.set(true); 1314 sleeper.wakeup(); 1315 1316 while (true) 1317 { 1318 final int stillRunning = runningThreads.get(); 1319 if (stillRunning <= 0) 1320 { 1321 break; 1322 } 1323 else 1324 { 1325 try 1326 { 1327 Thread.sleep(1L); 1328 } catch (final Exception e) {} 1329 } 1330 } 1331 } 1332 1333 1334 1335 /** 1336 * {@inheritDoc} 1337 */ 1338 @Override() 1339 public LinkedHashMap<String[],String> getExampleUsages() 1340 { 1341 final LinkedHashMap<String[],String> examples = 1342 new LinkedHashMap<>(StaticUtils.computeMapCapacity(2)); 1343 1344 String[] args = 1345 { 1346 "--hostname", "server.example.com", 1347 "--port", "389", 1348 "--bindDN", "uid=admin,dc=example,dc=com", 1349 "--bindPassword", "password", 1350 "--entryDN", "uid=user.[1-1000000],ou=People,dc=example,dc=com", 1351 "--attribute", "description", 1352 "--valueLength", "12", 1353 "--numThreads", "10" 1354 }; 1355 String description = 1356 "Test modify performance by randomly selecting entries across a set " + 1357 "of one million users located below 'ou=People,dc=example,dc=com' " + 1358 "with ten concurrent threads and replacing the values for the " + 1359 "description attribute with a string of 12 randomly-selected " + 1360 "lowercase alphabetic characters."; 1361 examples.put(args, description); 1362 1363 args = new String[] 1364 { 1365 "--generateSampleRateFile", "variable-rate-data.txt" 1366 }; 1367 description = 1368 "Generate a sample variable rate definition file that may be used " + 1369 "in conjunction with the --variableRateData argument. The sample " + 1370 "file will include comments that describe the format for data to be " + 1371 "included in this file."; 1372 examples.put(args, description); 1373 1374 return examples; 1375 } 1376}