分布式压测 ngrinder

install & run
run by docker
refrence
core code
- controller
- agent
修改jar中class
cluster mode

本文介绍分布式压测系统ngrinder及ngrinder工作原理 ngrinder源码解析

install & run

dowload ngrinder-controller-3.4.1.war
deploy ngrinder-controller-3.4.1.war
dowload agent http://{controller_host_name}/agent/download
run agent ./run_agent.sh
create script http://localhost:8080/script/
create test http://localhost:8080/perftest/new
run test

run by docker

./install_maven_lib.sh
mvn clean
mvn compile
mvn package -Dmaven.test.skip=true
./docker/prepare.sh
使用docker-compose 运行 docker/docker-compose.yml

refrence

https://github.com/naver/ngrinder/wiki/Installation-Guide
https://github.com/naver/ngrinder/wiki/Agent-Configuration-Guide
https://testerhome.com/topics/4225
http://blog.csdn.net/zhiaini06/article/details/45287663

core code

controller

PrefTestRunnable.java

    /**
     * Run the given test.
     * <p/>
     * If fails, it marks STOP_BY_ERROR in the given {@link PerfTest} status
     *
     * @param perfTest perftest instance;
     */
    public void doTest(final PerfTest perfTest) {
        SingleConsole singleConsole = null;
        try {
            singleConsole = startConsole(perfTest);
            // prepare distribute groovy maven projet build
            ScriptHandler prepareDistribution = perfTestService.prepareDistribution(perfTest);
            // generate the properties for agent include runtime properties
            GrinderProperties grinderProperties = perfTestService.getGrinderProperties(perfTest, prepareDistribution);
            // agent start with properties
            startAgentsOn(perfTest, grinderProperties, checkCancellation(singleConsole));
            // distribute files to agents
            distributeFileOn(perfTest, checkCancellation(singleConsole));
            // report path
            singleConsole.setReportPath(perfTestService.getReportFileDirectory(perfTest));
            // start test with properties
            runTestOn(perfTest, grinderProperties, checkCancellation(singleConsole));
        } catch (SingleConsoleCancellationException ex) {
            // In case of error, mark the occurs error on perftest.
            doCancel(perfTest, singleConsole);
            notifyFinish(perfTest, StopReason.CANCEL_BY_USER);
        } catch (Exception e) {
            // In case of error, mark the occurs error on perftest.
            LOG.error("Error while executing test: {} - {} ", perfTest.getTestIdentifier(), e.getMessage());
            LOG.debug("Stack Trace is : ", e);
            doTerminate(perfTest, singleConsole);
            notifyFinish(perfTest, StopReason.ERROR_WHILE_PREPARE);
        }
    }

PrefTestService.java

    /**
     * Prepare files for distribution. This method stores the files on the path
     * ${NGRINDER_HOME}/perftest/{test_id}/dist folder.
     *
     * @param perfTest perfTest
     * @return File location in which the perftest script and resources are distributed.
     */
    public ScriptHandler prepareDistribution(PerfTest perfTest) {
        File perfTestDistDirectory = getDistributionPath(perfTest);
        User user = perfTest.getCreatedUser();
        FileEntry scriptEntry = checkNotNull(fileEntryService.getOne(user, checkNotEmpty(perfTest.getScriptName(), "perfTest should have script name"), getSafe(perfTest.getScriptRevision())), "script should exist");
        // Get all files in the script path
        ScriptHandler handler = scriptHandlerFactory.getHandler(scriptEntry);

        ProcessingResultPrintStream processingResult = new ProcessingResultPrintStream(new ByteArrayOutputStream());
        // buildmaven project maven clean complie package
        handler.prepareDist(perfTest.getId(), user, scriptEntry, perfTestDistDirectory, config.getControllerProperties(), processingResult);
        // zip dist directory
        try {
            CompressionUtils.zip(perfTestDistDirectory);
        } catch (IOException e) {
            throw processException("Error while file zip perfTestDistDirectory.");
        }
        LOGGER.info("File write is completed in {}", perfTestDistDirectory);
        if (!processingResult.isSuccess()) {
            File logDir = new File(getLogFileDirectory(perfTest), "distribution_log.txt");
            try {
                FileUtils.writeByteArrayToFile(logDir, processingResult.getLogByteArray());
            } catch (IOException e) {
                noOp();
            }
            throw processException("Error while file distribution is prepared.");
        }
        return handler;
    }

PerfTestService.java line:560

  /**
   * Create {@link GrinderProperties} based on the passed {@link PerfTest}.
   *
   * @param perfTest      base data
   * @param scriptHandler scriptHandler
   * @return created {@link GrinderProperties} instance
   */
  public GrinderProperties getGrinderProperties(PerfTest perfTest, ScriptHandler scriptHandler) {
    try {
      // Use default properties first
      GrinderProperties grinderProperties = new GrinderProperties(config.getHome().getDefaultGrinderProperties());

      User user = perfTest.getCreatedUser();

      // Get all files in the script path
      String scriptName = perfTest.getScriptName();
      FileEntry userDefinedGrinderProperties = fileEntryService.getOne(user, FilenameUtils.concat(FilenameUtils.getPath(scriptName), DEFAULT_GRINDER_PROPERTIES), -1L);
      if (!config.isSecurityEnabled() && userDefinedGrinderProperties != null) {
        // Make the property overridden by user property.
        GrinderProperties userProperties = new GrinderProperties();
        userProperties.load(new StringReader(userDefinedGrinderProperties.getContent()));
        grinderProperties.putAll(userProperties);
      }
      grinderProperties.setAssociatedFile(new File(DEFAULT_GRINDER_PROPERTIES));
      grinderProperties.setProperty(GRINDER_PROP_SCRIPT, scriptHandler.getScriptExecutePath(scriptName));

      grinderProperties.setProperty(GRINDER_PROP_TEST_ID, "test_" + perfTest.getId());
      grinderProperties.setInt(GRINDER_PROP_AGENTS, getSafe(perfTest.getAgentCount()));
      grinderProperties.setInt(GRINDER_PROP_PROCESSES, getSafe(perfTest.getProcesses()));
      grinderProperties.setInt(GRINDER_PROP_THREAD, getSafe(perfTest.getThreads()));
      if (perfTest.isThresholdDuration()) {
        grinderProperties.setLong(GRINDER_PROP_DURATION, getSafe(perfTest.getDuration()));
        grinderProperties.setInt(GRINDER_PROP_RUNS, 0);
      } else {
        grinderProperties.setInt(GRINDER_PROP_RUNS, getSafe(perfTest.getRunCount()));
        if (grinderProperties.containsKey(GRINDER_PROP_DURATION)) {
          grinderProperties.remove(GRINDER_PROP_DURATION);
        }
      }
      grinderProperties.setProperty(GRINDER_PROP_ETC_HOSTS, StringUtils.defaultIfBlank(perfTest.getTargetHosts(), ""));
      grinderProperties.setBoolean(GRINDER_PROP_USE_CONSOLE, true);
      if (BooleanUtils.isTrue(perfTest.getUseRampUp())) {
        grinderProperties.setBoolean(GRINDER_PROP_THREAD_RAMPUP, perfTest.getRampUpType() == RampUp.THREAD);
        grinderProperties.setInt(GRINDER_PROP_PROCESS_INCREMENT, getSafe(perfTest.getRampUpStep()));
        grinderProperties.setInt(GRINDER_PROP_PROCESS_INCREMENT_INTERVAL, getSafe(perfTest.getRampUpIncrementInterval()));
        if (perfTest.getRampUpType() == RampUp.PROCESS) {
          grinderProperties.setInt(GRINDER_PROP_INITIAL_SLEEP_TIME, getSafe(perfTest.getRampUpInitSleepTime()));
        } else {
          grinderProperties.setInt(GRINDER_PROP_INITIAL_THREAD_SLEEP_TIME, getSafe(perfTest.getRampUpInitSleepTime()));
        }
        grinderProperties.setInt(GRINDER_PROP_INITIAL_PROCESS, getSafe(perfTest.getRampUpInitCount()));
      } else {
        grinderProperties.setInt(GRINDER_PROP_PROCESS_INCREMENT, 0);
      }
      grinderProperties.setInt(GRINDER_PROP_REPORT_TO_CONSOLE, 500);
      grinderProperties.setProperty(GRINDER_PROP_USER, perfTest.getCreatedUser().getUserId());
      // set grinder.jvm.classpath for agent run
      grinderProperties.setProperty(GRINDER_PROP_JVM_CLASSPATH, getCustomClassPath(perfTest));
      grinderProperties.setInt(GRINDER_PROP_IGNORE_SAMPLE_COUNT, getSafe(perfTest.getIgnoreSampleCount()));
      grinderProperties.setBoolean(GRINDER_PROP_SECURITY, config.isSecurityEnabled());
      // For backward agent compatibility.
      // If the security is not enabled, pass it as jvm argument.
      // If enabled, pass it to grinder.param. In this case, I drop the
      // compatibility.
      if (StringUtils.isNotBlank(perfTest.getParam())) {
        String param = perfTest.getParam().replace("'", "\\'").replace(" ", "");
        if (config.isSecurityEnabled()) {
          grinderProperties.setProperty(GRINDER_PROP_PARAM, StringUtils.trimToEmpty(param));
        } else {
          String property = grinderProperties.getProperty(GRINDER_PROP_JVM_ARGUMENTS, "");
          property = property + " -Dparam=" + param + " ";
          // set grinder.jvm.arguments for agent run
          grinderProperties.setProperty(GRINDER_PROP_JVM_ARGUMENTS, property);
        }
      }
      LOGGER.info("Grinder Properties : {} ", grinderProperties);
      return grinderProperties;
    } catch (Exception e) {
      throw processException("error while prepare grinder property for " + perfTest.getTestName(), e);
    }
  }

GroovyMavenProjectScriptHandler.java

@Override
    protected void prepareDistMore(Long testId, User user, FileEntry script, File distDir, PropertiesWrapper properties, ProcessingResultPrintStream processingResult) {
        String pomPathInSVN = PathUtils.join(getBasePath(script), "pom.xml");
        MavenCli cli = new MavenCli();
        processingResult.println("\nCopy dependencies by running 'mvn dependency:copy-dependencies"+ " -DoutputDirectory=./lib -DexcludeScope=provided'");
        // excute maven commond
        int result = cli.doMain(new String[]{ // goal specification
                "dependency:copy-dependencies", // run dependency goal
                "-DoutputDirectory=./lib", // to the lib folder
                "-DexcludeScope=provided" // but exclude the provided
                // library
        }, distDir.getAbsolutePath(), processingResult, processingResult);
        boolean success = (result == 0);
        if (success) {
            processingResult.printf("\nDependencies in %s was copied.\n", pomPathInSVN);
            LOGGER.info("Dependencies in {} is copied into {}/lib folder", pomPathInSVN, distDir.getAbsolutePath());
        } else {
            processingResult.printf("\nDependencies copy in %s is failed.\n", pomPathInSVN);
            LOGGER.info("Dependencies copy in {} is failed.", pomPathInSVN);
        }
        // Then it's not necessary to include pom.xml anymore.
        FileUtils.deleteQuietly(new File(distDir, "pom.xml"));
        processingResult.setSuccess(result == 0);
    }

PrefTestRunnable.java

    /**
     * Distribute files to agents.
     *
     * @param perfTest      perftest
     * @param singleConsole console to be used.
     */
    void distributeFileOn(final PerfTest perfTest, SingleConsole singleConsole) {
        // Distribute files
        perfTestService.markStatusAndProgress(perfTest, DISTRIBUTE_FILES, "All necessary files are being distributed.");
        ListenerSupport<SingleConsole.FileDistributionListener> listener = ListenerHelper.create();
        final long safeThreadHold = getSafeTransmissionThreshold();

        listener.add(new SingleConsole.FileDistributionListener() {
            @Override
            public void distributed(String fileName) {
                perfTestService.markProgress(perfTest, " - " + fileName);
            }

            @Override
            public boolean start(File dir, boolean safe) {
                if (safe) {
                    perfTestService.markProgress(perfTest, "Safe file distribution mode is enabled.");
                    return safe;
                }
                long sizeOfDirectory = FileUtils.sizeOfDirectory(dir);
                if (sizeOfDirectory > safeThreadHold) {
                    perfTestService.markProgress(perfTest, "The total size of distributed files is over "+ UnitUtils.byteCountToDisplaySize(safeThreadHold) + "B.\n- Safe file distribution mode is enabled by force.");
                    return true;
                }
                return safe;
            }

        });

        // the files have prepared before
        // start distribute files
        singleConsole.distributeFiles(perfTestService.getDistributionPath(perfTest), listener, isSafeDistPerfTest(perfTest));
        perfTestService.markStatusAndProgress(perfTest, DISTRIBUTE_FILES_FINISHED, "All necessary files are distributed.");
    }

SingleConsole.java

    /**
     * Distribute files on agents.
     *
     * @param listener listener
     * @param safe     safe mode
     */
    public void distributeFiles(ListenerSupport<FileDistributionListener> listener, final boolean safe) {
        final FileDistribution fileDistribution = getConsoleComponent(FileDistribution.class);
        final AgentCacheState agentCacheState = fileDistribution.getAgentCacheState();
        final Condition cacheStateCondition = new Condition();
        agentCacheState.addListener(new PropertyChangeListener() {
            public void propertyChange(PropertyChangeEvent ignored) {
                synchronized (cacheStateCondition) {
                    cacheStateCondition.notifyAll();
                }
            }
        });
        final MutableBoolean safeDist = new MutableBoolean(safe);
        ConsoleProperties consoleComponent = getConsoleComponent(ConsoleProperties.class);
        final File file = consoleComponent.getDistributionDirectory().getFile();
        if (listener != null) {
            listener.apply(new Informer<FileDistributionListener>() {
                @Override
                public void inform(FileDistributionListener listener) {
                    safeDist.setValue(listener.start(file, safe));
                }
            });
        }
        final FileDistributionHandler distributionHandler = fileDistribution.getHandler();
        // When cancel is called.. stop processing.
        int fileCount = 0;
        while (!cancel) {
            try {
                // distribute files in directory
                final FileDistributionHandler.Result result = distributionHandler.sendNextFile();
                fileCount++;
                if (result == null) {
                    break;
                }
                if (listener != null) {
                    listener.apply(new Informer<FileDistributionListener>() {
                        @Override
                        public void inform(FileDistributionListener listener) {
                            listener.distributed(result.getFileName());
                        }
                    });
                }

                if (safeDist.isTrue()) {
                    // The cache status is updated asynchronously by agent
                    // reports. If the listener is registered, this waits for up
                    // to five seconds for
                    // all agents to indicate that they are up to date.
                    checkSafetyWithCacheState(fileDistribution, cacheStateCondition, 1);
                }
            } catch (FileContents.FileContentsException e) {
                throw processException("Error while distribute files for " + getConsolePort());
            }
        }
        if (safeDist.isFalse()) {
            ThreadUtils.sleep(1000);
            checkSafetyWithCacheState(fileDistribution, cacheStateCondition, fileCount);
        }
    }

net/sf/grinder/grinder-core/3.9.1/grinder-core-3.9.1.jar!/net/grinder/console/communication/DistributionControlImplementation.class

  public Result sendNextFile() throws FileContents.FileContentsException {
    try {
      if (m_fileIndex < m_files.length) {
        if (m_fileIndex == 0) {
          // Clear any cache that has out of date cache parameters.
          // We currently we do nothing about cached copies of deleted files.
          final Address addressAgentsWithInvalidCaches = m_agents.getAddressOfOutOfDateAgents(0);

          m_distributionControl.clearFileCaches(addressAgentsWithInvalidCaches);
        }

        try {
          final int index = m_fileIndex;
          final File file = m_files[index];

          final Address addressAgentsWithoutFile = m_agents.getAddressOfOutOfDateAgents(new File(m_directory, file.getPath()).lastModified());
          // send one file to address of one agent
          m_distributionControl.sendFile(addressAgentsWithoutFile, new FileContents(m_directory, file));

          return new Result() {
              public int getProgressInCents() {
                return ((index + 1) * 100) / m_files.length;
              }

              public String getFileName() {
                return file.getPath();
              }
            };
        } finally {
          ++m_fileIndex;
        }
      } else {
        m_distributionControl.setHighWaterMark(
          m_agents.getAddressOfAllAgents(),
          m_cacheParameters.createHighWaterMark(m_latestFileTime));

        return null;
      }
    } catch (OutOfDateException e) {
      return null;
    }
  }