I wrote a simple PhantomJS script which automates a process on a website. A part of this process is submitting a form. I've been using this simple program for months using a quality proxy provider. Recently I switched providers and now I'm unable to bypass the form. Upon further testing (setting proxy on local computer) I found that using the proxy as socks rather than http allows me to bypass the form as needed.
However, when trying to set "--proxy-type=socks5" I get an error:
Network - Resource request error: QNetworkReply::NetworkError(UnknownNetworkError) ( "SOCKS version 5 protocol error" ) URL: "https://api.ipify.org/?format=json%27"
The command line I am trying to run is:
phantomjs --debug --proxy-type=socks5 --proxy=69.46.80.98:16151 ip.js
The content of ip.js:
var fs = require('fs');
var system = require('system');
var page = new WebPage(), testindex = 0, loadInProgress = false;
page.setReferrer = false;
var userAgents = [
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
'Mozilla/5.0 (Windows NT 5.1; rv:52.0) Gecko/20100101 Firefox/52.0'
];
var sessionAgent = userAgents[Math.floor(Math.random() * userAgents.length)];
page.settings.userAgent = sessionAgent;
page.settings.resourceTimeout = 600000;
page.settings.loadImages = false;
page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.onLoadStarted = function() {
loadInProgress = true;
console.log("load started");
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log("load finished");
};
var steps = [
function() {
page.open('https://api.ipify.org/?format=json%27');
},
function() {
console.log( page.plainText );
}
];
interval = setInterval(function() {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}, 50);
The full debugging output:
root#earth:~# phantomjs --debug=true --proxy-type=socks5 --proxy=69.46.80.98:16151 /home/phantomjs/downloaders/ip.js
2017-12-17T10:48:36 [DEBUG] CookieJar - Created but will not store cookies (use option '--cookies-file=<filename>' to enable persistent cookie storage)
2017-12-17T10:48:36 [DEBUG] Set "socks5" proxy to: "69.46.80.98" : 16151
2017-12-17T10:48:36 [DEBUG] Phantom - execute: Configuration
2017-12-17T10:48:36 [DEBUG] 0 objectName : ""
2017-12-17T10:48:36 [DEBUG] 1 cookiesFile : ""
2017-12-17T10:48:36 [DEBUG] 2 diskCacheEnabled : "false"
2017-12-17T10:48:36 [DEBUG] 3 maxDiskCacheSize : "-1"
2017-12-17T10:48:36 [DEBUG] 4 diskCachePath : ""
2017-12-17T10:48:36 [DEBUG] 5 ignoreSslErrors : "false"
2017-12-17T10:48:36 [DEBUG] 6 localUrlAccessEnabled : "true"
2017-12-17T10:48:36 [DEBUG] 7 localToRemoteUrlAccessEnabled : "false"
2017-12-17T10:48:36 [DEBUG] 8 outputEncoding : "UTF-8"
2017-12-17T10:48:36 [DEBUG] 9 proxyType : "socks5"
2017-12-17T10:48:36 [DEBUG] 10 proxy : "69.46.80.98:16151"
2017-12-17T10:48:36 [DEBUG] 11 proxyAuth : ":"
2017-12-17T10:48:36 [DEBUG] 12 scriptEncoding : "UTF-8"
2017-12-17T10:48:36 [DEBUG] 13 webSecurityEnabled : "true"
2017-12-17T10:48:36 [DEBUG] 14 offlineStoragePath : ""
2017-12-17T10:48:36 [DEBUG] 15 localStoragePath : ""
2017-12-17T10:48:36 [DEBUG] 16 localStorageDefaultQuota : "-1"
2017-12-17T10:48:36 [DEBUG] 17 offlineStorageDefaultQuota : "-1"
2017-12-17T10:48:36 [DEBUG] 18 printDebugMessages : "true"
2017-12-17T10:48:36 [DEBUG] 19 javascriptCanOpenWindows : "true"
2017-12-17T10:48:36 [DEBUG] 20 javascriptCanCloseWindows : "true"
2017-12-17T10:48:36 [DEBUG] 21 sslProtocol : "default"
2017-12-17T10:48:36 [DEBUG] 22 sslCiphers : "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-RC4-SHA:ECDHE-RSA-RC4-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:RC4-SHA:RC4-MD5"
2017-12-17T10:48:36 [DEBUG] 23 sslCertificatesPath : ""
2017-12-17T10:48:36 [DEBUG] 24 sslClientCertificateFile : ""
2017-12-17T10:48:36 [DEBUG] 25 sslClientKeyFile : ""
2017-12-17T10:48:36 [DEBUG] 26 sslClientKeyPassphrase : ""
2017-12-17T10:48:36 [DEBUG] 27 webdriver : ":"
2017-12-17T10:48:36 [DEBUG] 28 webdriverLogFile : ""
2017-12-17T10:48:36 [DEBUG] 29 webdriverLogLevel : "INFO"
2017-12-17T10:48:36 [DEBUG] 30 webdriverSeleniumGridHub : ""
2017-12-17T10:48:36 [DEBUG] Phantom - execute: Script & Arguments
2017-12-17T10:48:36 [DEBUG] script: "/home/phantomjs/downloaders/ip.js"
2017-12-17T10:48:36 [DEBUG] Phantom - execute: Starting normal mode
2017-12-17T10:48:36 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:36 [DEBUG] FileSystem - _open: ":/modules/fs.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:36 [DEBUG] FileSystem - _open: ":/modules/system.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:36 [DEBUG] FileSystem - _open: ":/modules/webpage.js" QMap(("mode", QVariant(QString, "r")))
step 1
2017-12-17T10:48:36 [DEBUG] WebPage - updateLoadingProgress: 10
load started
2017-12-17T10:48:37 [DEBUG] skipping hostname of len 13
2017-12-17T10:48:37 [DEBUG] Network - Resource request error: QNetworkReply::NetworkError(UnknownNetworkError) ( "SOCKS version 5 protocol error" ) URL: "https://api.ipify.org/?format=json%27"
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 100
load finished
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
step 2
test complete!
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 10
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 100
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/fs.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/system.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/webpage.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 10
2017-12-17T10:48:37 [DEBUG] WebPage - setupFrame ""
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/fs.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/system.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] FileSystem - _open: ":/modules/webpage.js" QMap(("mode", QVariant(QString, "r")))
2017-12-17T10:48:37 [DEBUG] WebPage - updateLoadingProgress: 100
If proxy is working this could be due to SSL errors. Try using these switches in command-line:
--ssl-protocol=any --ignore-ssl-errors
Related
My playbook used to be able to run successfully without any issues.
But recently, I ran the playbook again (without changing anything) and got this error:
TASK [read_csv] *****************************************************
fatal: [localhost]: FAILED! => {"changed": false, "msg": "Key 'username' is not unique for value ''"}
This is the playbook:
---
- name: Read User
hosts: localhost
tasks:
- read_csv:
path: opt/ansible.csv
key: username
fieldnames: username,usergroups,ipaddress,lastlogindate
delimiter: ','
register: usersdata
This is an example of the CSV file that I am using:
Username
User_groups
IP Address
Last Login Date
Apple
A
192.34.50.183
12/4/2021
Blizz
B
192.34.50.145
3/8/2021
What is value ''? And why is using 'username' as the key now wrong?
===============================================================
Update:
I tried the playbook suggested by Vladimir Botka below, and it was able to run successfully.
PLAY [Read User] ***************************************************************
TASK [Gathering Facts] *********************************************************
ok: [localhost]
TASK [read_csv] ****************************************************************
ok: [localhost]
TASK [debug] *******************************************************************
ok: [localhost] => {
"usersdata.dict": {}
}
TASK [debug] *******************************************************************
ok: [localhost] => {
"usersdata.list": [
{
"IP Address": "192.169.60.220 - XXX.XXX.60.254",
"Last Login Date": "30-Jul-21",
"User_groups": "B",
"Username": "Vijay"
},
{
"IP Address": "XXX.XXX.60.146 - XXX.XXX.60.147",
"Last Login Date": "1-Jan-21",
"User_groups": "ecquaria",
"Username": "wangjcecquaria"
},
…
TASK [read_csv] ****************************************************************
ok: [localhost]
TASK [debug] *******************************************************************
ok: [localhost] => {
"usersdata.dict": {
"Kyawryobi": {
"IP Address": "XXX.XXX.60.184 - XXX.XXX.60.185",
"Last Login Date": "28-Jul-21",
"User_groups": "Ryobi",
"Username": "Kyawryobi"
},
"Nikhil": {
"IP Address": "XXX.XXX.60.1 - XXX.XXX.60.80",
"Last Login Date": "30-Jul-21",
"User_groups": "A",
"Username": "Joshua"
},
"Vijay": {
…
TASK [debug] *******************************************************************
ok: [localhost] => {
"usersdata.list": []
}
TASK [read_csv] ****************************************************************
ok: [localhost]
TASK [debug] *******************************************************************
ok: [localhost] => {
"usersdata.dict": {}
}
TASK [debug] *******************************************************************
ok: [localhost] => {
"usersdata.list": [
{
"ipaddress": "IP Address",
"lastlogindate": "Last Login Date",
"usergroups": "User_groups",
"username": "Username"
},
{
"ipaddress": "192.169.60.220 - XXX.XXX.60.254",
"lastlogindate": "30-Jul-21",
"usergroups": "A",
"username": "Joshua"
},
{
…
TASK [read_csv] ****************************************************************
ok: [localhost]
TASK [debug] *******************************************************************
ok: [localhost] => {
"usersdata.dict": {
"Kyawryobi": {
"ipaddress": "XXX.XXX.60.184 - XXX.XXX.60.185",
"lastlogindate": "28-Jul-21",
"usergroups": "Ryobi",
"username": "Kyawryobi"
},
"Nikhil": {
"ipaddress": "XXX.XXX.60.1 - XXX.XXX.60.80",
"lastlogindate": "30-Jul-21",
"usergroups": "A",
"username": "Joshua"
},
"Username": {
…
TASK [debug] *******************************************************************
ok: [localhost] => {
"usersdata.list": []
}
PLAY RECAP *********************************************************************
localhost : ok=13 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
Next, I tried running my playbook again without changing anything else in the settings or playbook and there is no more error.
PLAY [Read User] ***************************************************************
TASK [Gathering Facts] *********************************************************
ok: [localhost]
TASK [read_csv] ****************************************************************
ok: [localhost]
PLAY RECAP *********************************************************************
localhost : ok=2 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
I think the error, "Key is not unique for value ''", might be the result of a bug, and running a playbook that triggers ansible to output a dictionary and list using the usersdata variable might have resolved it. I have checked and there is nothing wrong with the data in my CSV. There are no missing values in the 'Username' column, neither do I have the word 'username' duplicated anywhere else.
By default, the module community.general.read_csv takes the first line of a file as a header. You can declare fieldnames if the header is missing. Quoting:
A list of field names for every column. This is needed if the CSV does not have a header.
Given the file
shell> cat /tmp/ansible.csv
Username,User_groups,IP Address,Last Login Date
Apple,A,192.34.50.183,12/4/2021
Blizz,B,192.34.50.145,3/8/2021
You don't have to declare fieldnames because the first line is the header
- read_csv:
path: /tmp/ansible.csv
register: usersdata
gives (see the notes below on the output dictionary vs. list)
usersdata.list:
- IP Address: 192.34.50.183
Last Login Date: 12/4/2021
User_groups: A
Username: Apple
- IP Address: 192.34.50.145
Last Login Date: 3/8/2021
User_groups: B
Username: Blizz
If you do declare fieldnames all lines are treated as data
- read_csv:
path: /tmp/ansible.csv
fieldnames: username,usergroups,ipaddress,lastlogindate
register: usersdata
gives the list with renamed fields including an additional item of the original header
usersdata.list:
- ipaddress: IP Address
lastlogindate: Last Login Date
usergroups: User_groups
username: Username
- ipaddress: 192.34.50.183
lastlogindate: 12/4/2021
usergroups: A
username: Apple
- ipaddress: 192.34.50.145
lastlogindate: 3/8/2021
usergroups: B
username: Blizz
If you declare a key in addition to the fieldnames
- read_csv:
path: /tmp/ansible.csv
key: username
fieldnames: username,usergroups,ipaddress,lastlogindate
register: usersdata
you get the dictionary of this key's values
usersdata.dict:
Apple:
ipaddress: 192.34.50.183
lastlogindate: 12/4/2021
usergroups: A
username: Apple
Blizz:
ipaddress: 192.34.50.145
lastlogindate: 3/8/2021
usergroups: B
username: Blizz
Username:
ipaddress: IP Address
lastlogindate: Last Login Date
usergroups: User_groups
username: Username
Notes
The output depends on the parameter key.
a) Without a key
- read_csv:
path: /tmp/ansible.csv
register: usersdata
- debug:
var: usersdata.dict
- debug:
var: usersdata.list
you get an empty dictionary and a list of the lines
usersdata.dict: {}
usersdata.list:
- IP Address: 192.34.50.183
Last Login Date: 12/4/2021
User_groups: A
Username: Apple
- IP Address: 192.34.50.145
Last Login Date: 3/8/2021
User_groups: B
Username: Blizz
b) If you specify a key (it's case-sensitive)
- read_csv:
path: /tmp/ansible.csv
key: Username
register: usersdata
you get a dictionary of this key's values and an empty list
usersdata.dict:
Apple:
IP Address: 192.34.50.183
Last Login Date: 12/4/2021
User_groups: A
Username: Apple
Blizz:
IP Address: 192.34.50.145
Last Login Date: 3/8/2021
User_groups: B
Username: Blizz
usersdata.list: []
Example of a complete playbook for testing
- hosts: localhost
tasks:
- read_csv:
path: /tmp/ansible.csv
register: usersdata
- debug:
var: usersdata.dict
- debug:
var: usersdata.list
- read_csv:
path: /tmp/ansible.csv
key: Username
register: usersdata
- debug:
var: usersdata.dict
- debug:
var: usersdata.list
- read_csv:
path: /tmp/ansible.csv
fieldnames: username,usergroups,ipaddress,lastlogindate
register: usersdata
- debug:
var: usersdata.dict
- debug:
var: usersdata.list
- read_csv:
path: /tmp/ansible.csv
key: username
fieldnames: username,usergroups,ipaddress,lastlogindate
register: usersdata
- debug:
var: usersdata.dict
- debug:
var: usersdata.list
Project: SeleniumRCTestLibrary
System: Windows 7 Enterprise SP1 x64
Maven: Apache Maven 3.5.3
IDE: Eclipse 4.7.3a (Oxygen.3a)
Environment/System variables:
JAVA_HOME C:\Program Files\Java\jdk1.8.0_162
JAVA_PATH C:\Program Files\Java\jdk1.8.0_162\bin
classpath C:\Program Files\Java\jdk1.8.0_162\lib
path included ";C:\Program Files\Java\jdk1.8.0_162\bin"
Project structure:
SeleniumRCTestLibrary
|--src/main/java/com/dummy/jb/ (Own project libraries)
|--src/test/java/com/dummy/jb/ (Own Test Classes and Test Suites)
| |--Main/Specs/*.java (Own Test Classes)
| |--TestSuite/ (Own Test Suites)
| |--Tenant/
| |--ChannelX
| |--TenantChannel_PlatformEnv_TestSuiteName.java (Single Test Suite to run)
|--JRE System Library (jre1.8.0_162)
|--Maven Dependencies
|--Referenced Libraries
|--src/
|--lib/ (Other libraries)
|--target/
|--pom.xml (Maven configuration file)
Issue:
In Eclipse 4.7.3a (Oxygen.3a) the given test suite runs succesfully (if I start the suite in jUnit 4 mode: Right click > Run As > Run configuration > Test runner: jUnit 4 > run).
But no runs if I use my local maven installation via command line:
cd to project directory
Command "mvn
-Dtest=TenantChannel_PlatformEnv_TestSuiteName test"
or "mvn -Dtest=TenantChannel_PlatformEnv_TestSuiteName test -X"
Log: without parameter "-X"
[INFO] Scanning for projects...
[INFO]
[INFO] ---------------< com.dummy.jb:SeleniumRCTestLibrary >---------------
[INFO] Building SeleniumRCTestLibrary 0.0.1-SNAPSHOT
[INFO] --------------------------------[ pom ]---------------------------------
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 0.109 s
[INFO] Finished at: 2018-04-20T15:45:27+02:00
[INFO] ------------------------------------------------------------------------
Log: with parameter "-X"
Apache Maven 3.5.3 (3383c37e1f9e9b3bc3df5050c29c8aff9f295297; 2018-02-24T20:49:05+01:00)
Maven home: C:\Program Files\Maven\apache-maven-3.5.3\bin\..
Java version: 1.8.0_162, vendor: Oracle Corporation
Java home: C:\Program Files\Java\jdk1.8.0_162\jre
Default locale: en_US, platform encoding: Cp1252
OS name: "windows 7", version: "6.1", arch: "amd64", family: "windows"
[DEBUG] Created new class realm maven.api
[DEBUG] Importing foreign packages into class realm maven.api
[DEBUG] Imported: javax.annotation.* < plexus.core
[DEBUG] Imported: javax.annotation.security.* < plexus.core
[DEBUG] Imported: javax.enterprise.inject.* < plexus.core
[DEBUG] Imported: javax.enterprise.util.* < plexus.core
[DEBUG] Imported: javax.inject.* < plexus.core
[DEBUG] Imported: org.apache.maven.* < plexus.core
[DEBUG] Imported: org.apache.maven.artifact < plexus.core
[DEBUG] Imported: org.apache.maven.classrealm < plexus.core
[DEBUG] Imported: org.apache.maven.cli < plexus.core
[DEBUG] Imported: org.apache.maven.configuration < plexus.core
[DEBUG] Imported: org.apache.maven.exception < plexus.core
[DEBUG] Imported: org.apache.maven.execution < plexus.core
[DEBUG] Imported: org.apache.maven.execution.scope < plexus.core
[DEBUG] Imported: org.apache.maven.lifecycle < plexus.core
[DEBUG] Imported: org.apache.maven.model < plexus.core
[DEBUG] Imported: org.apache.maven.monitor < plexus.core
[DEBUG] Imported: org.apache.maven.plugin < plexus.core
[DEBUG] Imported: org.apache.maven.profiles < plexus.core
[DEBUG] Imported: org.apache.maven.project < plexus.core
[DEBUG] Imported: org.apache.maven.reporting < plexus.core
[DEBUG] Imported: org.apache.maven.repository < plexus.core
[DEBUG] Imported: org.apache.maven.rtinfo < plexus.core
[DEBUG] Imported: org.apache.maven.settings < plexus.core
[DEBUG] Imported: org.apache.maven.toolchain < plexus.core
[DEBUG] Imported: org.apache.maven.usability < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.* < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.authentication < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.authorization < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.events < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.observers < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.proxy < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.repository < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.resource < plexus.core
[DEBUG] Imported: org.codehaus.classworlds < plexus.core
[DEBUG] Imported: org.codehaus.plexus.* < plexus.core
[DEBUG] Imported: org.codehaus.plexus.classworlds < plexus.core
[DEBUG] Imported: org.codehaus.plexus.component < plexus.core
[DEBUG] Imported: org.codehaus.plexus.configuration < plexus.core
[DEBUG] Imported: org.codehaus.plexus.container < plexus.core
[DEBUG] Imported: org.codehaus.plexus.context < plexus.core
[DEBUG] Imported: org.codehaus.plexus.lifecycle < plexus.core
[DEBUG] Imported: org.codehaus.plexus.logging < plexus.core
[DEBUG] Imported: org.codehaus.plexus.personality < plexus.core
[DEBUG] Imported: org.codehaus.plexus.util.xml.Xpp3Dom < plexus.core
[DEBUG] Imported: org.codehaus.plexus.util.xml.pull.XmlPullParser < plexus.core
[DEBUG] Imported: org.codehaus.plexus.util.xml.pull.XmlPullParserException < plexus.core
[DEBUG] Imported: org.codehaus.plexus.util.xml.pull.XmlSerializer < plexus.core
[DEBUG] Imported: org.eclipse.aether.* < plexus.core
[DEBUG] Imported: org.eclipse.aether.artifact < plexus.core
[DEBUG] Imported: org.eclipse.aether.collection < plexus.core
[DEBUG] Imported: org.eclipse.aether.deployment < plexus.core
[DEBUG] Imported: org.eclipse.aether.graph < plexus.core
[DEBUG] Imported: org.eclipse.aether.impl < plexus.core
[DEBUG] Imported: org.eclipse.aether.installation < plexus.core
[DEBUG] Imported: org.eclipse.aether.internal.impl < plexus.core
[DEBUG] Imported: org.eclipse.aether.metadata < plexus.core
[DEBUG] Imported: org.eclipse.aether.repository < plexus.core
[DEBUG] Imported: org.eclipse.aether.resolution < plexus.core
[DEBUG] Imported: org.eclipse.aether.spi < plexus.core
[DEBUG] Imported: org.eclipse.aether.transfer < plexus.core
[DEBUG] Imported: org.eclipse.aether.version < plexus.core
[DEBUG] Imported: org.fusesource.jansi.* < plexus.core
[DEBUG] Imported: org.slf4j.* < plexus.core
[DEBUG] Imported: org.slf4j.helpers.* < plexus.core
[DEBUG] Imported: org.slf4j.spi.* < plexus.core
[DEBUG] Populating class realm maven.api
[INFO] Error stacktraces are turned on.
[DEBUG] Message scheme: color
[DEBUG] Message styles: debug info warning error success failure strong mojo project
[DEBUG] Reading global settings from C:\Program Files\Maven\apache-maven-3.5.3\bin\..\conf\settings.xml
[DEBUG] Reading user settings from C:\Users\cnorris\.m2\settings.xml
[DEBUG] Reading global toolchains from C:\Program Files\Maven\apache-maven-3.5.3\bin\..\conf\toolchains.xml
[DEBUG] Reading user toolchains from C:\Users\cnorris\.m2\toolchains.xml
[DEBUG] Using local repository at C:\Users\cnorris\.m2\repository
[DEBUG] Using manager EnhancedLocalRepositoryManager with priority 10.0 for C:\Users\cnorris\.m2\repository
[INFO] Scanning for projects...
[DEBUG] Extension realms for project com.dummy.jb:SeleniumRCTestLibrary:pom:0.0.1-SNAPSHOT: (none)
[DEBUG] Looking up lifecycle mappings for packaging pom from ClassRealm[plexus.core, parent: null]
[DEBUG] === REACTOR BUILD PLAN ================================================
[DEBUG] Project: com.dummy.jb:SeleniumRCTestLibrary:pom:0.0.1-SNAPSHOT
[DEBUG] Tasks: [test]
[DEBUG] Style: Regular
[DEBUG] =======================================================================
[INFO]
[INFO] ---------------< com.dummy.jb:SeleniumRCTestLibrary >---------------
[INFO] Building SeleniumRCTestLibrary 0.0.1-SNAPSHOT
[INFO] --------------------------------[ pom ]---------------------------------
[DEBUG] Lifecycle default -> [validate, initialize, generate-sources, process-sources, generate-resources, process-resources, compile, process-classes, generate-test-sources, process-test-sources, generate-test-resources, process-test-resources, test-compile, process-test-classes, test, prepare-package, package, pre-integration-test, integration-test, post-integration-test, verify, install, deploy]
[DEBUG] Lifecycle clean -> [pre-clean, clean, post-clean]
[DEBUG] Lifecycle site -> [pre-site, site, post-site, site-deploy]
[DEBUG] === PROJECT BUILD PLAN ================================================
[DEBUG] Project: com.dummy.jb:SeleniumRCTestLibrary:0.0.1-SNAPSHOT
[DEBUG] Dependencies (collect): []
[DEBUG] Dependencies (resolve): []
[DEBUG] Repositories (dependencies): [central (https://repo.maven.apache.org/maven2, default, releases)]
[DEBUG] Repositories (plugins) : [central (https://repo.maven.apache.org/maven2, default, releases)]
[DEBUG] =======================================================================
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 0.125 s
[INFO] Finished at: 2018-04-20T15:46:18+02:00
[INFO] ------------------------------------------------------------------------
Minimal project files, including the pom.xml file:
Click to download
I think the maven environment find not the given test suite name.
In eclipse it works fine.
I want to use nginx for rate limiting and caching.
In which order nginx applies them? In other words, is it limiting only request to the upstream server or all requests (including cache HIT's)?
How can this order be changed? I think it can be changed by having two server contexts. So, for example, on one server performs caching. It has the second server context as upstream. The second one limits requests to the "real" upstream. But that's probably not the most efficient way...
Request Processing
Nginx request processing is is done with a number of different phases, with each phase having one or more handlers. Modules can register to run at a specific phase.
http://www.nginxguts.com/phases/
http://nginx.org/en/docs/dev/development_guide.html#http_phases
Rate Limiting
Rate limiting is applied by the ngx_http_limit_req_module in the pre access phase. From ngx_http_limit_req_module.c:
h = ngx_array_push(&cmcf->phases[NGX_HTTP_PREACCESS_PHASE].handlers);
Caching
Caching is done later, I think in the content phase.
I couldn't quite figure this out from looking at the code or the documentation. But I was able to demonstrate this with a debug build. My configuration had rate limiting 1 request per second, with caching on. See the following excerpts from my log.
Cached Request
...
2020/08/01 11:11:07 [debug] 17498#0: *7 http header done
2020/08/01 11:11:07 [debug] 17498#0: *7 rewrite phase: 0
2020/08/01 11:11:07 [debug] 17498#0: *7 test location: "/"
2020/08/01 11:11:07 [debug] 17498#0: *7 using configuration "=/"
2020/08/01 11:11:07 [debug] 17498#0: *7 http cl:-1 max:1048576
2020/08/01 11:11:07 [debug] 17498#0: *7 rewrite phase: 2
2020/08/01 11:11:07 [debug] 17498#0: *7 post rewrite phase: 3
2020/08/01 11:11:07 [debug] 17498#0: *7 generic phase: 4
2020/08/01 11:11:07 [debug] 17498#0: *7 http script var: ....
2020/08/01 11:11:07 [debug] 17498#0: shmtx lock
2020/08/01 11:11:07 [debug] 17498#0: shmtx unlock
2020/08/01 11:11:07 [debug] 17498#0: *7 limit_req[0]: 0 0.000
2020/08/01 11:11:07 [debug] 17498#0: *7 generic phase: 5
2020/08/01 11:11:07 [debug] 17498#0: *7 access phase: 6
2020/08/01 11:11:07 [debug] 17498#0: *7 access phase: 7
2020/08/01 11:11:07 [debug] 17498#0: *7 post access phase: 8
2020/08/01 11:11:07 [debug] 17498#0: *7 generic phase: 9
2020/08/01 11:11:07 [debug] 17498#0: *7 generic phase: 10
2020/08/01 11:11:07 [debug] 17498#0: *7 http init upstream, client timer: 0
2020/08/01 11:11:07 [debug] 17498#0: *7 http cache key: "http://127.0.0.1:9000"
2020/08/01 11:11:07 [debug] 17498#0: *7 http cache key: "/"
2020/08/01 11:11:07 [debug] 17498#0: *7 add cleanup: 00005609F7C51578
2020/08/01 11:11:07 [debug] 17498#0: shmtx lock
2020/08/01 11:11:07 [debug] 17498#0: shmtx unlock
2020/08/01 11:11:07 [debug] 17498#0: *7 http file cache exists: 0 e:1
2020/08/01 11:11:07 [debug] 17498#0: *7 cache file: "/home/poida/src/nginx-1.15.6/objs/cache/157d4d91f488c05ff417723d74d65b36"
2020/08/01 11:11:07 [debug] 17498#0: *7 add cleanup: 00005609F7C46810
2020/08/01 11:11:07 [debug] 17498#0: *7 http file cache fd: 12
2020/08/01 11:11:07 [debug] 17498#0: *7 read: 12, 00005609F7C46890, 519, 0
2020/08/01 11:11:07 [debug] 17498#0: *7 http upstream cache: 0
2020/08/01 11:11:07 [debug] 17498#0: *7 http proxy status 200 "200 OK"
2020/08/01 11:11:07 [debug] 17498#0: *7 http proxy header: "Server: SimpleHTTP/0.6 Python/3.8.5"
2020/08/01 11:11:07 [debug] 17498#0: *7 http proxy header: "Date: Sat, 01 Aug 2020 01:11:03 GMT"
2020/08/01 11:11:07 [debug] 17498#0: *7 http proxy header: "Content-type: text/html; charset=utf-8"
2020/08/01 11:11:07 [debug] 17498#0: *7 http proxy header: "Content-Length: 340"
2020/08/01 11:11:07 [debug] 17498#0: *7 http proxy header done
2020/08/01 11:11:07 [debug] 17498#0: *7 http file cache send: /home/poida/src/nginx-1.15.6/objs/cache/157d4d91f488c05ff417723d74d65b36
2020/08/01 11:11:07 [debug] 17498#0: *7 posix_memalign: 00005609F7C46DC0:4096 #16
2020/08/01 11:11:07 [debug] 17498#0: *7 HTTP/1.1 200 OK
...
Rate Limited Request
...
2020/08/01 11:17:04 [debug] 17498#0: *10 http header done
2020/08/01 11:17:04 [debug] 17498#0: *10 rewrite phase: 0
2020/08/01 11:17:04 [debug] 17498#0: *10 test location: "/"
2020/08/01 11:17:04 [debug] 17498#0: *10 using configuration "=/"
2020/08/01 11:17:04 [debug] 17498#0: *10 http cl:-1 max:1048576
2020/08/01 11:17:04 [debug] 17498#0: *10 rewrite phase: 2
2020/08/01 11:17:04 [debug] 17498#0: *10 post rewrite phase: 3
2020/08/01 11:17:04 [debug] 17498#0: *10 generic phase: 4
2020/08/01 11:17:04 [debug] 17498#0: *10 http script var: ....
2020/08/01 11:17:04 [debug] 17498#0: shmtx lock
2020/08/01 11:17:04 [debug] 17498#0: shmtx unlock
2020/08/01 11:17:04 [debug] 17498#0: *10 limit_req[0]: -3 0.707
2020/08/01 11:17:04 [error] 17498#0: *10 limiting requests, excess: 0.707 by zone "mylimit", client: 127.0.0.1, server: , request: "GET / HTTP/1.1", host: "localhost:8080"
2020/08/01 11:17:04 [debug] 17498#0: *10 http finalize request: 503, "/?" a:1, c:1
2020/08/01 11:17:04 [debug] 17498#0: *10 http special response: 503, "/?"
2020/08/01 11:17:04 [debug] 17498#0: *10 http set discard body
2020/08/01 11:17:04 [debug] 17498#0: *10 HTTP/1.1 503 Service Temporarily Unavailable
...
For a rate limited request, processing stops before the server tries to generate the content or check the cache.
TL;DR; Rate limiting is applied first, before caching.
I have faced the same case: if the rate limiter rejects a request then return an already cached response.
In this case you can use error_page directive and "direct" to another location.
proxy_cache_path /var/cache/nginx/html_cache levels=1:2 use_temp_path=off keys_zone=html_cache:128m max_size=5G inactive=7d;
limit_req_zone $binary_remote_addr zone=req_per_ip_limit:64m rate=1r/m;
proxy_cache_key $request_method|$host|$request_uri;
location #html-cache-fallback {
proxy_cache html_cache;
proxy_cache_use_stale error timeout invalid_header http_500 http_502 http_503 http_504 http_403 http_404 http_429;
add_header x-debug-rate-limit REJECTED;
# Set nonexistent host in order to receive error and then return response from cache
proxy_pass http://0.0.0.0:7777;
}
location / {
limit_req zone=req_per_ip_limit;
limit_req_status 429;
# In case rate limit rejects request direct to another location
error_page 429 =200 #html-cache-fallback;
# Cache response
proxy_cache html_cache;
proxy_cache_valid 200 1s;
proxy_cache_use_stale error timeout invalid_header http_500 http_502 http_503 http_504 http_403 http_404 http_429;
proxy_pass https://example.com;
}
If you have network load balancer then you have to implement rate and caching on both server.
Rate limit is like this.
location /login/ {
limit_req zone=mylimit burst=20;
proxy_pass http://my_upstream;
}
The burst parameter defines how many requests a client can make in excess of the rate specified by the zone (with our sample mylimit zone, the rate limit is 10 requests per second, or 1 every 100 milliseconds). A request that arrives sooner than 100 milliseconds after the previous one is put in a queue, and here we are setting the queue size to 20.
That means if 21 requests arrive from a given IP address simultaneously, NGINX forwards the first one to the upstream server group immediately and puts the remaining 20 in the queue. It then forwards a queued request every 100 milliseconds, and returns 503 to the client only if an incoming request makes the number of queued requests go over 20.
Now if you do limit rating on one server only then two subsequent request which goes to different server will have issue.
All caching variable needs to be synced too. you need Redis or persistent storage for caching.
https://www.nginx.com/blog/rate-limiting-nginx/
NGINX
How does Nginx rate limiter works
NGINX rate limiting uses the leaky bucket algorithm, which is widely
used in telecommunications and packet‑switched computer networks to
deal with burstiness when bandwidth is limited. The analogy is with a
bucket where water is poured in at the top and leaks from the bottom;
if the rate at which water is poured in exceeds the rate at which it
leaks, the bucket overflows. In terms of request processing, the water
represents requests from clients, and the bucket represents a queue
where requests wait to be processed according to a first‑in‑first‑out
(FIFO) scheduling algorithm. The leaking water represents requests
exiting the buffer for processing by the server, and the overflow
represents requests that are discarded and never serviced.
Adding snap of config from one of my servers:
The ngx_http_limit_conn_module module is used to limit the number of connections per the defined key, in particular, the number of connections from a single IP address.
Not all connections are counted. A connection is counted only if it has a request being processed by the server and the whole request header has already been read.
So basically you can do two setups for actual and for all other individual virtual servers.
By limiting IP
By limiting Connection
There could be several limit_conn directives. For example, the
following configuration will limit the number of connections to the
server per client IP and, at the same time, the total number of
connections to the virtual server:
Below is the example for the same
limit_conn_zone $binary_remote_addr zone=perip:10m;
limit_conn_zone $server_name zone=perserver:10m;
server {
...
limit_conn perip 10;
limit_conn perserver 100;
}
Together ahead!
Darpan
You can find in detail example and explanation here too
I'm using Maven 3.0.4 and want to deploy something in my local repository to a remote repository, to which I've verified I have access. I'm using the below command …
mvn -X deploy:deploy-file -DgroupId=org.directwebremoting -DartifactId=dwr -Dversion=3.0.0-rc2 -Dpackaging=jar -Dfile=/Users/davea/.m2/repository//org/directwebremoting/dwr/3.0.0-rc2/dwr-3.0.0-rc2.jar -Durl=dav:https://repository-myco.forge.cloudbees.com/private -DrepositoryId=cloudbees-private
but am getting this unhelpful error message. Any ideas what else I should check? Following the error is my ~.m2/settings.xml file.
Apache Maven 3.0.4 (r1232337; 2012-01-17 02:44:56-0600)
Maven home: /opt/apache-maven-3.0.4
Java version: 1.6.0_37, vendor: Apple Inc.
Java home: /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Home
Default locale: en_US, platform encoding: MacRoman
OS name: "mac os x", version: "10.7.5", arch: "x86_64", family: "mac"
[INFO] Error stacktraces are turned on.
[DEBUG] Reading global settings from /opt/apache-maven-3.0.4/conf/settings.xml
[DEBUG] Reading user settings from /Users/davea/.m2/settings.xml
[DEBUG] Using local repository at /Users/davea/.m2/repository
[DEBUG] Using manager EnhancedLocalRepositoryManager with priority 10 for /Users/davea/.m2/repository
[INFO] Scanning for projects...
[DEBUG] Extension realms for project org.mainco.subco:myproject:war:1.0-SNAPSHOT: (none)
[DEBUG] Looking up lifecyle mappings for packaging war from ClassRealm[plexus.core, parent: null]
[DEBUG] Resolving plugin prefix deploy from [org.apache.maven.plugins, org.codehaus.mojo]
[DEBUG] Resolved plugin prefix deploy to org.apache.maven.plugins:maven-deploy-plugin from POM org.mainco.subco:myproject:war:1.0-SNAPSHOT
[DEBUG] === REACTOR BUILD PLAN ================================================
[DEBUG] Project: org.mainco.subco:myproject:war:1.0-SNAPSHOT
[DEBUG] Tasks: [deploy:deploy-file]
[DEBUG] Style: Aggregating
[DEBUG] =======================================================================
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building myproject-war 1.0-SNAPSHOT
[INFO] ------------------------------------------------------------------------
[DEBUG] Resolving plugin prefix deploy from [org.apache.maven.plugins, org.codehaus.mojo]
[DEBUG] Resolved plugin prefix deploy to org.apache.maven.plugins:maven-deploy-plugin from POM org.mainco.subco:myproject:war:1.0-SNAPSHOT
[DEBUG] Lifecycle default -> [validate, initialize, generate-sources, process-sources, generate-resources, process-resources, compile, process-classes, generate-test-sources, process-test-sources, generate-test-resources, process-test-resources, test-compile, process-test-classes, test, prepare-package, package, pre-integration-test, integration-test, post-integration-test, verify, install, deploy]
[DEBUG] Lifecycle clean -> [pre-clean, clean, post-clean]
[DEBUG] Lifecycle site -> [pre-site, site, post-site, site-deploy]
[DEBUG] === PROJECT BUILD PLAN ================================================
[DEBUG] Project: org.mainco.subco:myproject:1.0-SNAPSHOT
[DEBUG] Dependencies (collect): []
[DEBUG] Dependencies (resolve): []
[DEBUG] Repositories (dependencies): [cloudbees-private (https://repository-myco.forge.cloudbees.com/private/, releases+snapshots), maven2-repository.dev.java.net (http://download.java.net/maven/2/, releases), central (http://repo.maven.apache.org/maven2, releases)]
[DEBUG] Repositories (plugins) : [repository.jboss.org_thirdparty-releases (https://repository.jboss.org/nexus/content/repositories/thirdparty-releases, releases), repository.jboss.org_thirdparty-uploads (https://repository.jboss.org/nexus/content/repositories/thirdparty-uploads, releases), central (http://repo.maven.apache.org/maven2, releases)]
[DEBUG] -----------------------------------------------------------------------
[DEBUG] Goal: org.apache.maven.plugins:maven-deploy-plugin:2.7:deploy-file (default-cli)
[DEBUG] Style: Regular
[DEBUG] Configuration: <?xml version="1.0" encoding="UTF-8"?>
<configuration>
<artifactId>${artifactId}</artifactId>
<classifier>${classifier}</classifier>
<classifiers>${classifiers}</classifiers>
<description>${generatePom.description}</description>
<file>${file}</file>
<files>${files}</files>
<generatePom default-value="true">${generatePom}</generatePom>
<groupId>${groupId}</groupId>
<javadoc>${javadoc}</javadoc>
<localRepository default-value="${localRepository}"/>
<offline default-value="${settings.offline}"/>
<packaging>${packaging}</packaging>
<pomFile>${pomFile}</pomFile>
<project default-value="${project}"/>
<repositoryId default-value="remote-repository">${repositoryId}</repositoryId>
<repositoryLayout default-value="default">${repositoryLayout}</repositoryLayout>
<retryFailedDeploymentCount default-value="1">${retryFailedDeploymentCount}</retryFailedDeploymentCount>
<sources>${sources}</sources>
<types>${types}</types>
<uniqueVersion default-value="true">${uniqueVersion}</uniqueVersion>
<updateReleaseInfo default-value="false">${updateReleaseInfo}</updateReleaseInfo>
<url>${url}</url>
<version>${version}</version>
</configuration>
[DEBUG] =======================================================================
[INFO]
[INFO] --- maven-deploy-plugin:2.7:deploy-file (default-cli) # myproject ---
[DEBUG] Created new class realm maven.api
[DEBUG] Importing foreign packages into class realm maven.api
[DEBUG] Imported: org.apache.maven.wagon.events < plexus.core
[DEBUG] Imported: org.sonatype.aether.transfer < plexus.core
[DEBUG] Imported: org.apache.maven.exception < plexus.core
[DEBUG] Imported: org.sonatype.aether.metadata < plexus.core
[DEBUG] Imported: org.codehaus.plexus.util.xml.Xpp3Dom < plexus.core
[DEBUG] Imported: org.sonatype.aether.collection < plexus.core
[DEBUG] Imported: org.sonatype.aether.version < plexus.core
[DEBUG] Imported: org.apache.maven.monitor < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.repository < plexus.core
[DEBUG] Imported: org.apache.maven.repository < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.resource < plexus.core
[DEBUG] Imported: org.codehaus.plexus.logging < plexus.core
[DEBUG] Imported: org.apache.maven.profiles < plexus.core
[DEBUG] Imported: org.sonatype.aether.repository < plexus.core
[DEBUG] Imported: org.apache.maven.classrealm < plexus.core
[DEBUG] Imported: org.apache.maven.execution < plexus.core
[DEBUG] Imported: org.sonatype.aether.artifact < plexus.core
[DEBUG] Imported: org.sonatype.aether.spi < plexus.core
[DEBUG] Imported: org.apache.maven.reporting < plexus.core
[DEBUG] Imported: org.apache.maven.usability < plexus.core
[DEBUG] Imported: org.codehaus.plexus.container < plexus.core
[DEBUG] Imported: org.codehaus.plexus.component < plexus.core
[DEBUG] Imported: org.codehaus.plexus.util.xml.pull.XmlSerializer < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.authentication < plexus.core
[DEBUG] Imported: org.apache.maven.lifecycle < plexus.core
[DEBUG] Imported: org.codehaus.plexus.classworlds < plexus.core
[DEBUG] Imported: org.sonatype.aether.graph < plexus.core
[DEBUG] Imported: org.sonatype.aether.* < plexus.core
[DEBUG] Imported: org.apache.maven.settings < plexus.core
[DEBUG] Imported: org.codehaus.classworlds < plexus.core
[DEBUG] Imported: org.sonatype.aether.impl < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.* < plexus.core
[DEBUG] Imported: org.apache.maven.toolchain < plexus.core
[DEBUG] Imported: org.sonatype.aether.deployment < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.observers < plexus.core
[DEBUG] Imported: org.codehaus.plexus.util.xml.pull.XmlPullParserException < plexus.core
[DEBUG] Imported: org.codehaus.plexus.util.xml.pull.XmlPullParser < plexus.core
[DEBUG] Imported: org.apache.maven.configuration < plexus.core
[DEBUG] Imported: org.apache.maven.cli < plexus.core
[DEBUG] Imported: org.sonatype.aether.installation < plexus.core
[DEBUG] Imported: org.codehaus.plexus.context < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.authorization < plexus.core
[DEBUG] Imported: org.apache.maven.project < plexus.core
[DEBUG] Imported: org.apache.maven.rtinfo < plexus.core
[DEBUG] Imported: org.codehaus.plexus.lifecycle < plexus.core
[DEBUG] Imported: org.codehaus.plexus.configuration < plexus.core
[DEBUG] Imported: org.apache.maven.artifact < plexus.core
[DEBUG] Imported: org.apache.maven.model < plexus.core
[DEBUG] Imported: org.apache.maven.* < plexus.core
[DEBUG] Imported: org.apache.maven.wagon.proxy < plexus.core
[DEBUG] Imported: org.sonatype.aether.resolution < plexus.core
[DEBUG] Imported: org.apache.maven.plugin < plexus.core
[DEBUG] Imported: org.codehaus.plexus.* < plexus.core
[DEBUG] Imported: org.codehaus.plexus.personality < plexus.core
[DEBUG] Populating class realm maven.api
[DEBUG] org.apache.maven.plugins:maven-deploy-plugin:jar:2.7:
[DEBUG] org.apache.maven:maven-plugin-api:jar:2.0.6:compile
[DEBUG] org.apache.maven:maven-project:jar:2.0.6:compile
[DEBUG] org.apache.maven:maven-settings:jar:2.0.6:compile
[DEBUG] org.apache.maven:maven-profile:jar:2.0.6:compile
[DEBUG] org.apache.maven:maven-artifact-manager:jar:2.0.6:compile
[DEBUG] org.apache.maven:maven-repository-metadata:jar:2.0.6:compile
[DEBUG] org.apache.maven:maven-plugin-registry:jar:2.0.6:compile
[DEBUG] org.codehaus.plexus:plexus-container-default:jar:1.0-alpha-9-stable-1:compile
[DEBUG] junit:junit:jar:3.8.1:compile
[DEBUG] classworlds:classworlds:jar:1.1-alpha-2:compile
[DEBUG] org.apache.maven:maven-model:jar:2.0.6:compile
[DEBUG] org.apache.maven:maven-artifact:jar:2.0.6:compile
[DEBUG] org.codehaus.plexus:plexus-utils:jar:1.5.6:compile
[DEBUG] Created new class realm plugin>org.apache.maven.plugins:maven-deploy-plugin:2.7
[DEBUG] Importing foreign packages into class realm plugin>org.apache.maven.plugins:maven-deploy-plugin:2.7
[DEBUG] Imported: < maven.api
[DEBUG] Populating class realm plugin>org.apache.maven.plugins:maven-deploy-plugin:2.7
[DEBUG] Included: org.apache.maven.plugins:maven-deploy-plugin:jar:2.7
[DEBUG] Included: junit:junit:jar:3.8.1
[DEBUG] Included: org.codehaus.plexus:plexus-utils:jar:1.5.6
[DEBUG] Excluded: org.apache.maven:maven-plugin-api:jar:2.0.6
[DEBUG] Excluded: org.apache.maven:maven-project:jar:2.0.6
[DEBUG] Excluded: org.apache.maven:maven-settings:jar:2.0.6
[DEBUG] Excluded: org.apache.maven:maven-profile:jar:2.0.6
[DEBUG] Excluded: org.apache.maven:maven-artifact-manager:jar:2.0.6
[DEBUG] Excluded: org.apache.maven:maven-repository-metadata:jar:2.0.6
[DEBUG] Excluded: org.apache.maven:maven-plugin-registry:jar:2.0.6
[DEBUG] Excluded: org.codehaus.plexus:plexus-container-default:jar:1.0-alpha-9-stable-1
[DEBUG] Excluded: classworlds:classworlds:jar:1.1-alpha-2
[DEBUG] Excluded: org.apache.maven:maven-model:jar:2.0.6
[DEBUG] Excluded: org.apache.maven:maven-artifact:jar:2.0.6
[DEBUG] Configuring mojo org.apache.maven.plugins:maven-deploy-plugin:2.7:deploy-file from plugin realm ClassRealm[plugin>org.apache.maven.plugins:maven-deploy-plugin:2.7, parent: sun.misc.Launcher$AppClassLoader#69cd2e5f]
[DEBUG] Configuring mojo 'org.apache.maven.plugins:maven-deploy-plugin:2.7:deploy-file' with basic configurator -->
[DEBUG] (f) artifactId = dwr
[DEBUG] (f) file = /Users/davea/.m2/repository/org/directwebremoting/dwr/3.0.0-rc2/dwr-3.0.0-rc2.jar
[DEBUG] (f) generatePom = true
[DEBUG] (f) groupId = org.directwebremoting
[DEBUG] (s) localRepository = id: local
url: file:///Users/davea/.m2/repository/
layout: none
[DEBUG] (f) offline = false
[DEBUG] (f) packaging = jar
[DEBUG] (f) project = MavenProject: org.mainco.subco:myproject:1.0-SNAPSHOT # /Users/davea/Dropbox/workspace/myproject/pom.xml
[DEBUG] (f) repositoryId = cloudbees-private
[DEBUG] (f) repositoryLayout = default
[DEBUG] (f) retryFailedDeploymentCount = 1
[DEBUG] (f) uniqueVersion = true
[DEBUG] (f) updateReleaseInfo = false
[DEBUG] (f) url = dav:https://repository-myco.forge.cloudbees.com/private
[DEBUG] (f) version = 3.0.0-rc2
[DEBUG] -- end configuration --
[INFO] ------------------------------------------------------------------------
[INFO] BUILD FAILURE
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 0.662s
[INFO] Finished at: Tue Jan 08 14:02:33 CST 2013
[INFO] Final Memory: 4M/81M
[INFO] ------------------------------------------------------------------------
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:2.7:deploy-file (default-cli) on project myproject: Cannot deploy artifact from the local repository: /Users/davea/.m2/repository/org/directwebremoting/dwr/3.0.0-rc2/dwr-3.0.0-rc2.jar -> [Help 1]
org.apache.maven.lifecycle.LifecycleExecutionException: Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:2.7:deploy-file (default-cli) on project myproject: Cannot deploy artifact from the local repository: /Users/davea/.m2/repository/org/directwebremoting/dwr/3.0.0-rc2/dwr-3.0.0-rc2.jar
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:213)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:153)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:145)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:84)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:59)
at org.apache.maven.lifecycle.internal.LifecycleStarter.singleThreadedBuild(LifecycleStarter.java:183)
at org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:161)
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:320)
at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:156)
at org.apache.maven.cli.MavenCli.execute(MavenCli.java:537)
at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:196)
at org.apache.maven.cli.MavenCli.main(MavenCli.java:141)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:290)
at org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:230)
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:409)
at org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:352)
Caused by: org.apache.maven.plugin.MojoFailureException: Cannot deploy artifact from the local repository: /Users/davea/.m2/repository/org/directwebremoting/dwr/3.0.0-rc2/dwr-3.0.0-rc2.jar
at org.apache.maven.plugin.deploy.DeployFileMojo.execute(DeployFileMojo.java:283)
at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:101)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:209)
... 19 more
[ERROR]
[ERROR]
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException
My ~.m2/settings.xml file
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<profiles>
<profile>
<id>default</id>
<repositories>
<repository>
<id>cloudbees-private</id>
<name>Cloudbees private repo</name>
<url>https://repository-myco.forge.cloudbees.com/private/</url>
</repository>
</repositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>default</activeProfile>
</activeProfiles>
<servers>
<server>
<username>username</username>
<password>password</password>
<id>cloudbees-private</id>
</server>
</servers>
</settings>
I don't know why, but deploy-file gives that error when you supply artifact path installed in the local repository. (More precisely when the file parameter is the exact location that artifact would have in the local repository).
http://grepcode.com/file/repo1.maven.org/maven2/org.apache.maven.plugins/maven-gpg-plugin/1.5/org/apache/maven/plugin/gpg/SignAndDeployFileMojo.java/#326
Moving the artifact file (normaly the jar) somewhere else and passing this path as the file parameter, solves the problem.
Starting from version 2.5 the deploy plugin checks if the file to upload resides in the local repository and refuses the upload if that’s the case. You could specify version 2.4 of the plugin as a work-around:
mvn org.apache.maven.plugins:maven-deploy-plugin:2.4:deploy-file -DgroupId=org.directwebremoting -DartifactId=dwr -Dversion=3.0.0-rc2 -Dpackaging=jar -Dfile=/Users/davea/.m2/repository//org/directwebremoting/dwr/3.0.0-rc2/dwr-3.0.0-rc2.jar -Durl=dav:https://repository-myco.forge.cloudbees.com/private -DrepositoryId=cloudbees-private
I just ran into this whilst trying to migrate a load of dependencies from one private repository to another.
My solution was to simply make a soft link to the repository location (e.g. using ln -s) and reference the artifacts through that instead.
Hmm, I remember trying to deploy a file to my Cloudbees repository and it wouldn't work either. At the time, I queried Cloudbees about it through their forums, and one of their support folks deployed it in for me.
They do have some doco about it
http://wiki.cloudbees.com/bin/view/DEV/CloudBees+Private+Maven+Repository
I think it's more about how the CLoudbees server accepts the requests, rather than you doing anything wrong.
I would suggest to reverse your approach. Download it locally and deploy to the repository manager straight away from the downloaded file and then do NOT install into the local repository, but rather as a first test see if it proxies fine from the repo manager.
That will improve you flow by adding a test as well as provide a work around for the restriction of the deploy plugin and save you the execution of the install-file goal.
I am not certain why that restriction is part of the plugin but I suspect it has something to do with the meta data about the component in the local repository as compared to in a repository manager.
The wagon-webdav connector is needed. Create a temporary POM file to provide that information:
<project>
<modelVersion>4.0.0</modelVersion>
<groupId>dummy</groupId>
<artifactId>dummy</artifactId>
<version>1</version>
<build>
<extensions>
<extension>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-webdav</artifactId>
<version>1.0-beta-2</version>
</extension>
</extensions>
</build>
</project>
I'm trying tu use jboss-as-maven-plugin at pre(and post)-integration-test phase to deploy (and undeploy) my test webapp and when mvn install is executed, the (un-)deployment does not happen. The following Jenkins job's output seems to point me to configuration but I can't figure out whats's missing :
maven output :
mojoSucceeded org.apache.maven.plugins:maven-jar-plugin:2.3.2(default-jar)
forkedProjectSucceeded com.hck.debate:debate-test:2.1.15-SNAPSHOT
Dec 3, 2012 11:28:19 PM hudson.maven.ExecutedMojo <init>
WARNING: Failed to getClass for org.jboss.as.plugin.deployment.Deploy
mojoStarted org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final(set-env)
[INFO]
[INFO] --- jboss-as-maven-plugin:7.2.Final:deploy (set-env) # debate-test ---
Dec 3, 2012 11:28:20 PM org.xnio.Xnio <clinit>
INFO: XNIO Version 3.0.3.GA
Dec 3, 2012 11:28:20 PM org.xnio.nio.NioXnio <clinit>
INFO: XNIO NIO Implementation Version 3.0.3.GA
Dec 3, 2012 11:28:20 PM org.jboss.remoting3.EndpointImpl <clinit>
INFO: JBoss Remoting version 3.2.7.GA
mojoSucceeded org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final(set-env)
mojoStarted org.apache.maven.plugins:maven-failsafe-plugin:2.12.4(default)
maven output in debug mode (mvn -X install) :
mojoSucceeded org.apache.maven.plugins:maven-jar-plugin:2.3.2(default-jar)
forkedProjectSucceeded com.hck.debate:debate-test:2.1.16-SNAPSHOT
Dec 4, 2012 6:00:45 PM hudson.maven.ExecutedMojo <init>
WARNING: Failed to getClass for org.jboss.as.plugin.deployment.Deploy
mojoStarted org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final(set-env)
[INFO]
[INFO] --- jboss-as-maven-plugin:7.2.Final:deploy (set-env) # debate-test ---
[DEBUG] org.jboss.as.plugins:jboss-as-maven-plugin:jar:7.2.Final:
[DEBUG] org.apache.maven:maven-plugin-api:jar:3.0.4:compile
[DEBUG] org.apache.maven:maven-model:jar:3.0.4:compile
[DEBUG] org.apache.maven:maven-artifact:jar:3.0.4:compile
[DEBUG] org.sonatype.sisu:sisu-inject-plexus:jar:2.3.0:compile
[DEBUG] org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile
[DEBUG] org.codehaus.plexus:plexus-classworlds:jar:2.4:compile
[DEBUG] org.sonatype.sisu:sisu-inject-bean:jar:2.3.0:compile
[DEBUG] org.sonatype.sisu:sisu-guice:jar:no_aop:3.1.0:compile
[DEBUG] org.sonatype.sisu:sisu-guava:jar:0.9.9:compile
[DEBUG] org.apache.maven:maven-project:jar:2.2.1:compile
[DEBUG] org.apache.maven:maven-settings:jar:2.2.1:compile
[DEBUG] org.apache.maven:maven-profile:jar:2.2.1:compile
[DEBUG] org.apache.maven:maven-artifact-manager:jar:2.2.1:compile
[DEBUG] org.apache.maven:maven-repository-metadata:jar:2.2.1:compile
[DEBUG] backport-util-concurrent:backport-util-concurrent:jar:3.1:compile
[DEBUG] org.apache.maven:maven-plugin-registry:jar:2.2.1:compile
[DEBUG] org.codehaus.plexus:plexus-interpolation:jar:1.11:compile
[DEBUG] org.codehaus.plexus:plexus-container-default:jar:1.0-alpha-9-stable-1:compile
[DEBUG] junit:junit:jar:3.8.1:compile
[DEBUG] classworlds:classworlds:jar:1.1-alpha-2:compile
[DEBUG] org.sonatype.aether:aether-api:jar:1.8:compile
[DEBUG] org.sonatype.aether:aether-util:jar:1.8:compile
[DEBUG] org.jboss.as:jboss-as-cli:jar:7.1.2.Final:compile
[DEBUG] jline:jline:jar:0.9.94:compile
[DEBUG] org.jboss:staxmapper:jar:1.1.0.Final:compile
[DEBUG] org.jboss.logging:jboss-logging:jar:3.1.1.GA:compile
[DEBUG] org.jboss.sasl:jboss-sasl:jar:1.0.1.Final:compile
[DEBUG] org.jboss.remoting3:jboss-remoting:jar:3.2.7.GA:compile
[DEBUG] org.jboss.xnio:xnio-api:jar:3.0.3.GA:compile
[DEBUG] org.jboss.as:jboss-as-controller:jar:7.1.2.Final:compile
[DEBUG] org.jboss.modules:jboss-modules:jar:1.1.2.GA:compile
[DEBUG] org.jboss.msc:jboss-msc:jar:1.0.2.GA:compile
[DEBUG] org.jboss.remotingjmx:remoting-jmx:jar:1.0.3.Final:compile
[DEBUG] org.jboss.logmanager:jboss-logmanager:jar:1.3.0.Final:compile
[DEBUG] org.jboss.logging:jboss-logging-processor:jar:1.0.2.Final:compile
[DEBUG] system:jdk-tools:jar:jdk:system
[DEBUG] org.jboss.marshalling:jboss-marshalling:jar:1.3.14.GA:compile
[DEBUG] org.jboss.marshalling:jboss-marshalling-river:jar:1.3.14.GA:compile
[DEBUG] org.jboss.xnio:xnio-nio:jar:3.0.3.GA:compile
[DEBUG] org.jboss:jboss-vfs:jar:3.1.0.Final:compile
[DEBUG] sun.jdk:jconsole:jar:jdk:system
[DEBUG] org.jboss.as:jboss-as-build-config:jar:7.1.2.Final:compile
[DEBUG] org.jboss.as:jboss-as-controller-client:jar:7.1.2.Final:compile
[DEBUG] org.jboss.as:jboss-as-protocol:jar:7.1.2.Final:compile
[DEBUG] org.jboss:jboss-dmr:jar:1.1.1.Final:compile
[DEBUG] org.jboss.threads:jboss-threads:jar:2.0.0.GA:compile
[DEBUG] org.apache.maven.plugin-tools:maven-plugin-annotations:jar:3.1:compile
[DEBUG] org.codehaus.plexus:plexus-utils:jar:3.0.1:compile
[DEBUG] Created new class realm plugin>org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final
[DEBUG] Importing foreign packages into class realm plugin>org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final
[DEBUG] Imported: < maven.api
[DEBUG] Populating class realm plugin>org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final
[DEBUG] Included: org.jboss.as.plugins:jboss-as-maven-plugin:jar:7.2.Final
[DEBUG] Included: org.codehaus.plexus:plexus-component-annotations:jar:1.5.5
[DEBUG] Included: org.sonatype.sisu:sisu-inject-bean:jar:2.3.0
[DEBUG] Included: org.sonatype.sisu:sisu-guice:jar:no_aop:3.1.0
[DEBUG] Included: org.sonatype.sisu:sisu-guava:jar:0.9.9
[DEBUG] Included: backport-util-concurrent:backport-util-concurrent:jar:3.1
[DEBUG] Included: org.codehaus.plexus:plexus-interpolation:jar:1.11
[DEBUG] Included: junit:junit:jar:3.8.1
[DEBUG] Included: org.sonatype.aether:aether-util:jar:1.8
[DEBUG] Included: org.jboss.as:jboss-as-cli:jar:7.1.2.Final
[DEBUG] Included: jline:jline:jar:0.9.94
[DEBUG] Included: org.jboss:staxmapper:jar:1.1.0.Final
[DEBUG] Included: org.jboss.logging:jboss-logging:jar:3.1.1.GA
[DEBUG] Included: org.jboss.sasl:jboss-sasl:jar:1.0.1.Final
[DEBUG] Included: org.jboss.remoting3:jboss-remoting:jar:3.2.7.GA
[DEBUG] Included: org.jboss.xnio:xnio-api:jar:3.0.3.GA
[DEBUG] Included: org.jboss.as:jboss-as-controller:jar:7.1.2.Final
[DEBUG] Included: org.jboss.modules:jboss-modules:jar:1.1.2.GA
[DEBUG] Included: org.jboss.msc:jboss-msc:jar:1.0.2.GA
[DEBUG] Included: org.jboss.remotingjmx:remoting-jmx:jar:1.0.3.Final
[DEBUG] Included: org.jboss.logmanager:jboss-logmanager:jar:1.3.0.Final
[DEBUG] Included: org.jboss.logging:jboss-logging-processor:jar:1.0.2.Final
[DEBUG] Included: system:jdk-tools:jar:jdk
[DEBUG] Included: org.jboss.marshalling:jboss-marshalling:jar:1.3.14.GA
[DEBUG] Included: org.jboss.marshalling:jboss-marshalling-river:jar:1.3.14.GA
[DEBUG] Included: org.jboss.xnio:xnio-nio:jar:3.0.3.GA
[DEBUG] Included: org.jboss:jboss-vfs:jar:3.1.0.Final
[DEBUG] Included: sun.jdk:jconsole:jar:jdk
[DEBUG] Included: org.jboss.as:jboss-as-build-config:jar:7.1.2.Final
[DEBUG] Included: org.jboss.as:jboss-as-controller-client:jar:7.1.2.Final
[DEBUG] Included: org.jboss.as:jboss-as-protocol:jar:7.1.2.Final
[DEBUG] Included: org.jboss:jboss-dmr:jar:1.1.1.Final
[DEBUG] Included: org.jboss.threads:jboss-threads:jar:2.0.0.GA
[DEBUG] Included: org.apache.maven.plugin-tools:maven-plugin-annotations:jar:3.1
[DEBUG] Included: org.codehaus.plexus:plexus-utils:jar:3.0.1
[DEBUG] Excluded: org.apache.maven:maven-plugin-api:jar:3.0.4
[DEBUG] Excluded: org.apache.maven:maven-model:jar:3.0.4
[DEBUG] Excluded: org.apache.maven:maven-artifact:jar:3.0.4
[DEBUG] Excluded: org.sonatype.sisu:sisu-inject-plexus:jar:2.3.0
[DEBUG] Excluded: org.codehaus.plexus:plexus-classworlds:jar:2.4
[DEBUG] Excluded: org.apache.maven:maven-project:jar:2.2.1
[DEBUG] Excluded: org.apache.maven:maven-settings:jar:2.2.1
[DEBUG] Excluded: org.apache.maven:maven-profile:jar:2.2.1
[DEBUG] Excluded: org.apache.maven:maven-artifact-manager:jar:2.2.1
[DEBUG] Excluded: org.apache.maven:maven-repository-metadata:jar:2.2.1
[DEBUG] Excluded: org.apache.maven:maven-plugin-registry:jar:2.2.1
[DEBUG] Excluded: org.codehaus.plexus:plexus-container-default:jar:1.0-alpha-9-stable-1
[DEBUG] Excluded: classworlds:classworlds:jar:1.1-alpha-2
[DEBUG] Excluded: org.sonatype.aether:aether-api:jar:1.8
[DEBUG] Configuring mojo org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final:deploy from plugin realm ClassRealm[plugin>org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final, parent: sun.misc.Launcher$AppClassLoader#cac268]
[DEBUG] Configuring mojo 'org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final:deploy' with basic configurator -->
[DEBUG] (f) checkPackaging = true
[DEBUG] (f) force = true
[DEBUG] (f) hostname = 10.10.101.84
[DEBUG] (f) port = 9999
[DEBUG] (f) project = MavenProject: com.hck.debate:debate-test:2.1.16-SNAPSHOT # /var/lib/jenkins/jobs/sauce-dev/workspace/debate-test/pom.xml
[DEBUG] (f) skip = false
[DEBUG] (f) targetDir = /var/lib/jenkins/jobs/sauce-dev/workspace/debate-test/target
[DEBUG] -- end configuration --
Dec 4, 2012 6:00:46 PM org.xnio.Xnio <clinit>
INFO: XNIO Version 3.0.3.GA
Dec 4, 2012 6:00:46 PM org.xnio.nio.NioXnio <clinit>
INFO: XNIO NIO Implementation Version 3.0.3.GA
Dec 4, 2012 6:00:46 PM org.jboss.remoting3.EndpointImpl <clinit>
INFO: JBoss Remoting version 3.2.7.GA
[DEBUG] Executing deployment
mojoSucceeded org.jboss.as.plugins:jboss-as-maven-plugin:7.2.Final(set-env)
mojoStarted org.apache.maven.plugins:maven-failsafe-plugin:2.12.4(default)
my pom file :
<build>
<plugins>
<plugin>
<groupId>org.jboss.as.plugins</groupId>
<artifactId>jboss-as-maven-plugin</artifactId>
<version>7.2.Final</version>
<configuration>
<hostname>10.10.101.84</hostname>
<port>9999</port>
</configuration>
<executions>
<execution>
<id>set-env</id>
<phase>pre-integration-test</phase>
<configuration>
<force>true</force>
</configuration>
<goals>
<goal>deploy</goal>
</goals>
</execution>
<execution>
<id>unset-env</id>
<phase>post-integration-test</phase>
<goals>
<goal>undeploy</goal>
</goals>
</execution>
</executions>
<plugin>
</plugins>
</build>
Any clues?
Edit : I think I have a bigger issue here as jboss-as-maven-plugin execution is done in the test module (the pom file above is the test modules's one) so it tries to deploy the test module jar (and I want to deploy the war which is another module)...
I was trying to deploy the test module jar not the war (which is another module). Thanks to #James R. Perkins who helped me open my eyes