Cloud load testing tool with crawl capability - performance

Question

I need to get a glance of pages that are bottlenecks on a certain website. I was trying to find a cloud load testing tool that would start from homepage and descend 1 level into all links, but I couldn't find such tool.
Basically, what I need is to insert URL, and the script then tests with like 10 or 20 users every page that is linked from the page I inserted. That would provide me with a basic view of load times for most pages on the website.
Anyone familiar with such a tool?
Closest thing I could find was jcrawl, but it requires installing jakarta ant, server configuration and what not, and I think what I need is pretty basic and shouldn't require several hours to be put into scripting and configuration.
Thanks

Answer 1

You could achieve this using JMeter (JMeter is arguably the best free and open source load testing tool available). You could make one request and use a regular expression extractor to parse the response, pulling out any href values, and then have a loop which cycles through each url found.
Below is a JMeter testplan that does exactly what you want. Just copy this xml into a text file and save it as something.jmx. Open it in JMeter and change the demo hostname to YOUR OWN SITE (don't spam others, it's bad form). CTRL-R to run.
For more users (threads) or iterations (loops) just click on the Thread Group Name and change the values to what you need.
Very vague advice: Don't use more than, say, 300 threads on a single PC.
If you want to run this on 'the cloud' then rent an Amazon AWS machine, install jmeter, copy this jmx file onto it, and run it from the command line using:
$ cd /to/where/jmeter/is/apache-blah/bin/
$ ./jmeter -n -t /path/to/my/testplan.jmx -l /where/to/put/results.jtl
Demo Crawl Testplan:
<?xml version="1.0" encoding="UTF-8"?>
<jmeterTestPlan version="1.2" properties="2.3">
<hashTree>
<TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="crawl" enabled="true">
<stringProp name="TestPlan.comments"></stringProp>
<boolProp name="TestPlan.functional_mode">false</boolProp>
<boolProp name="TestPlan.serialize_threadgroups">false</boolProp>
<elementProp name="TestPlan.user_defined_variables" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true">
<collectionProp name="Arguments.arguments"/>
</elementProp>
<stringProp name="TestPlan.user_define_classpath"></stringProp>
</TestPlan>
<hashTree>
<CookieManager guiclass="CookiePanel" testclass="CookieManager" testname="HTTP Cookie Manager" enabled="true">
<collectionProp name="CookieManager.cookies"/>
<boolProp name="CookieManager.clearEachIteration">false</boolProp>
</CookieManager>
<hashTree/>
<ThreadGroup guiclass="ThreadGroupGui" testclass="ThreadGroup" testname="Thread Group Name" enabled="true">
<stringProp name="ThreadGroup.on_sample_error">startnextloop</stringProp>
<elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller" enabled="true">
<boolProp name="LoopController.continue_forever">false</boolProp>
<stringProp name="LoopController.loops">1</stringProp>
</elementProp>
<stringProp name="ThreadGroup.num_threads">1</stringProp>
<stringProp name="ThreadGroup.ramp_time">1</stringProp>
<longProp name="ThreadGroup.start_time">1346172541000</longProp>
<longProp name="ThreadGroup.end_time">1346172541000</longProp>
<boolProp name="ThreadGroup.scheduler">false</boolProp>
<stringProp name="ThreadGroup.duration"></stringProp>
<stringProp name="ThreadGroup.delay"></stringProp>
</ThreadGroup>
<hashTree>
<HTTPSamplerProxy guiclass="HttpTestSampleGui" testclass="HTTPSamplerProxy" testname="homepage" enabled="true">
<elementProp name="HTTPsampler.Arguments" elementType="Arguments" guiclass="HTTPArgumentsPanel" testclass="Arguments" enabled="true">
<collectionProp name="Arguments.arguments"/>
</elementProp>
<stringProp name="HTTPSampler.domain">www.bbc.co.uk</stringProp>
<stringProp name="HTTPSampler.port"></stringProp>
<stringProp name="HTTPSampler.connect_timeout"></stringProp>
<stringProp name="HTTPSampler.response_timeout"></stringProp>
<stringProp name="HTTPSampler.protocol"></stringProp>
<stringProp name="HTTPSampler.contentEncoding"></stringProp>
<stringProp name="HTTPSampler.path">/</stringProp>
<stringProp name="HTTPSampler.method">GET</stringProp>
<boolProp name="HTTPSampler.follow_redirects">true</boolProp>
<boolProp name="HTTPSampler.auto_redirects">false</boolProp>
<boolProp name="HTTPSampler.use_keepalive">true</boolProp>
<boolProp name="HTTPSampler.DO_MULTIPART_POST">false</boolProp>
<boolProp name="HTTPSampler.monitor">false</boolProp>
<stringProp name="HTTPSampler.embedded_url_re"></stringProp>
<stringProp name="TestPlan.comments">enter YOUR SITE in the Server Name field</stringProp>
</HTTPSamplerProxy>
<hashTree>
<RegexExtractor guiclass="RegexExtractorGui" testclass="RegexExtractor" testname="urls" enabled="true">
<stringProp name="RegexExtractor.useHeaders">false</stringProp>
<stringProp name="RegexExtractor.refname">urls</stringProp>
<stringProp name="RegexExtractor.regex"><a href="http:\/\/www.bbc.co.uk([^"]+)"</stringProp>
<stringProp name="RegexExtractor.template">$1$</stringProp>
<stringProp name="RegexExtractor.default">NOTFOUND</stringProp>
<stringProp name="RegexExtractor.match_number">-1</stringProp>
</RegexExtractor>
<hashTree/>
<ConstantThroughputTimer guiclass="TestBeanGUI" testclass="ConstantThroughputTimer" testname="Constant Throughput Timer" enabled="true">
<doubleProp>
<name>throughput</name>
<value>6.0</value>
<savedValue>0.0</savedValue>
</doubleProp>
<stringProp name="calcMode">this thread only</stringProp>
<stringProp name="TestPlan.comments">Each thread makes 6 requests every minute (if it completes in time)</stringProp>
</ConstantThroughputTimer>
<hashTree/>
</hashTree>
<ForeachController guiclass="ForeachControlPanel" testclass="ForeachController" testname="ForEach Controller" enabled="true">
<stringProp name="ForeachController.inputVal">urls</stringProp>
<stringProp name="ForeachController.returnVal">url</stringProp>
<boolProp name="ForeachController.useSeparator">true</boolProp>
</ForeachController>
<hashTree>
<HTTPSamplerProxy guiclass="HttpTestSampleGui" testclass="HTTPSamplerProxy" testname="foundUrl" enabled="true">
<elementProp name="HTTPsampler.Arguments" elementType="Arguments" guiclass="HTTPArgumentsPanel" testclass="Arguments" enabled="true">
<collectionProp name="Arguments.arguments"/>
</elementProp>
<stringProp name="HTTPSampler.domain">www.bbc.co.uk</stringProp>
<stringProp name="HTTPSampler.port"></stringProp>
<stringProp name="HTTPSampler.connect_timeout"></stringProp>
<stringProp name="HTTPSampler.response_timeout"></stringProp>
<stringProp name="HTTPSampler.protocol"></stringProp>
<stringProp name="HTTPSampler.contentEncoding"></stringProp>
<stringProp name="HTTPSampler.path">${url}</stringProp>
<stringProp name="HTTPSampler.method">GET</stringProp>
<boolProp name="HTTPSampler.follow_redirects">true</boolProp>
<boolProp name="HTTPSampler.auto_redirects">false</boolProp>
<boolProp name="HTTPSampler.use_keepalive">true</boolProp>
<boolProp name="HTTPSampler.DO_MULTIPART_POST">false</boolProp>
<boolProp name="HTTPSampler.monitor">false</boolProp>
<stringProp name="HTTPSampler.embedded_url_re"></stringProp>
<stringProp name="TestPlan.comments">enter YOUR SITE in the Server Name field</stringProp>
</HTTPSamplerProxy>
<hashTree>
<GaussianRandomTimer guiclass="GaussianRandomTimerGui" testclass="GaussianRandomTimer" testname="Gaussian Random Timer" enabled="true">
<stringProp name="ConstantTimer.delay">300</stringProp>
<stringProp name="RandomTimer.range">500.0</stringProp>
</GaussianRandomTimer>
<hashTree/>
</hashTree>
</hashTree>
</hashTree>
<ResultCollector guiclass="ViewResultsFullVisualizer" testclass="ResultCollector" testname="View Results Tree" enabled="true">
<boolProp name="ResultCollector.error_logging">false</boolProp>
<objProp>
<name>saveConfig</name>
<value class="SampleSaveConfiguration">
<time>true</time>
<latency>true</latency>
<timestamp>true</timestamp>
<success>true</success>
<label>true</label>
<code>true</code>
<message>true</message>
<threadName>true</threadName>
<dataType>true</dataType>
<encoding>false</encoding>
<assertions>true</assertions>
<subresults>true</subresults>
<responseData>false</responseData>
<samplerData>false</samplerData>
<xml>false</xml>
<fieldNames>true</fieldNames>
<responseHeaders>false</responseHeaders>
<requestHeaders>false</requestHeaders>
<responseDataOnError>false</responseDataOnError>
<saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
<assertionsResultsToSave>0</assertionsResultsToSave>
<bytes>true</bytes>
<hostname>true</hostname>
<threadCounts>true</threadCounts>
</value>
</objProp>
<stringProp name="filename"></stringProp>
</ResultCollector>
<hashTree/>
<ResultCollector guiclass="StatVisualizer" testclass="ResultCollector" testname="Aggregate Report" enabled="true">
<boolProp name="ResultCollector.error_logging">false</boolProp>
<objProp>
<name>saveConfig</name>
<value class="SampleSaveConfiguration">
<time>true</time>
<latency>true</latency>
<timestamp>true</timestamp>
<success>true</success>
<label>true</label>
<code>true</code>
<message>true</message>
<threadName>true</threadName>
<dataType>true</dataType>
<encoding>false</encoding>
<assertions>true</assertions>
<subresults>true</subresults>
<responseData>false</responseData>
<samplerData>false</samplerData>
<xml>false</xml>
<fieldNames>true</fieldNames>
<responseHeaders>false</responseHeaders>
<requestHeaders>false</requestHeaders>
<responseDataOnError>false</responseDataOnError>
<saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
<assertionsResultsToSave>0</assertionsResultsToSave>
<bytes>true</bytes>
<hostname>true</hostname>
<threadCounts>true</threadCounts>
</value>
</objProp>
<stringProp name="filename"></stringProp>
</ResultCollector>
<hashTree/>
</hashTree>
</hashTree>
</jmeterTestPlan>