Merge pull request #29 from fixy/prod

prod branch update

MANIFEST.MF

@@ -1,12 +1,22 @@
 Manifest-Version: 1.0
-Ant-Version: Apache Ant 1.9.6
-Created-By: 1.8.0_144-b01 (Oracle Corporation)
+Class-Path: io-2.2.0.jar bcprov-jdk15on-157.jar gss-2.2.0.jar gram-2.2.0.jar jsse-2.2.0.jar
+ axisg-2.2.0.jar gridftp-2.2.0.jar myproxy-2.2.0.jar commons-io-2.4.jar
+  commons-logging.jar commons-codec-1.7.jar commons-lang3-3.1.jar ssl-
+ proxies-2.2.0.jar io-2.2.0.jar gss-2.2.0.jar gram-2.2.0.jar jsse-2.2.
+ 0.jar axisg-2.2.0.jar gridftp-2.2.0.jar myproxy-2.2.0.jar commons-io-
+ 2.4.jar commons-logging.jar commons-codec-1.7.jar commons-lang3-3.1.j
+ ar ssl-proxies-2.2.0.jar BCGSS.jar ShiFT.jar SSHVnc.jar filedrop.jar 
+ libbrowser.jar log4j-1.2.6.jar SSHTerm-1.0.0.jar jlirc-unix-soc.jar p
+ utty-pk-1.1.0.jar SecureTunneling.jar bcprov-jdk16-146.jar j2ssh-core
+ -0.2.7.jar ncsa-lcrypto-146.jar TransferAPIClient.jar j2ssh-common-0.
+ 2.7.jar swing-layout-1.0.3.jar voms-api-java-2.0.9.jar commons-compre
+ ss-1.2.jar not-yet-commons-ssl-0.3.11.jar BCGSS.jar ShiFT.jar SSHVnc.
+ jar filedrop.jar libbrowser.jar log4j-1.2.6.jar SSHTerm-1.0.0.jar jli
+ rc-unix-soc.jar putty-pk-1.1.0.jar SecureTunneling.jar j2ssh-core-0.2
+ .7.jar ncsa-lcrypto-146.jar TransferAPIClient.jar j2ssh-common-0.2.7.
+ jar swing-layout-1.0.3.jar voms-api-java-2.0.9.jar commons-compress-1
+ .2.jar not-yet-commons-ssl-0.3.11.jar io-2.2.0.jar gss-2.2.0.jar gram
+ -2.2.0.jar jsse-2.2.0.jar axisg-2.2.0.jar gridftp-2.2.0.jar myproxy-2
+ .2.0.jar commons-io-2.4.jar commons-logging.jar commons-codec-1.7.jar
+  commons-lang3-3.1.jar ssl-proxies-2.2.0.jar bcprov-jdk15on-157.jar
 Main-Class: lia.util.net.copy.FDTMain
-Built-Jdk: 1.8.0_144-b01
-Built-By: rsirvins
-Built-Arch: amd64
-Built-Os: Linux 4.4.0-91-generic
-Release-Date: 2017-10-14
-Release-Version: 0.25.1
-Implementation-Version: 0.25.1-201704201041
-Class-Path: 

add-lib-to-local-maven.sh

@@ -0,0 +1,37 @@
+# Adding lib directory contents to a local maven repository then later we could use them in fdt building
+# Maven should be installed already on machine (i.e. use sudo apt-get install maven)
+
+#Globus dependencies
+mvn install:install-file -Dfile="lib/globus/gss-2.2.0.jar" -DgroupId=org.globus.gsi.gssapi -DartifactId=gss -Dversion=2.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/globus/io-2.2.0.jar" -DgroupId=org.globus.io -DartifactId=io -Dversion=2.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/globus/jsse-2.2.0.jar" -DgroupId=org.globus.jsse -DartifactId=jsse -Dversion=2.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/globus/myproxy-2.2.0.jar" -DgroupId=org.globus.myproxy -DartifactId=myproxy -Dversion=2.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/globus/ssl-proxies-2.2.0.jar" -DgroupId=org.globus -DartifactId=ssl-proxies -Dversion=2.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/globus/gridftp-2.2.0.jar" -DgroupId=org.globus -DartifactId=gridftp -Dversion=2.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/globus/gram-2.2.0.jar" -DgroupId=org.globus -DartifactId=gram -Dversion=2.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/globus/axisg-2.2.0.jar" -DgroupId=org.globus -DartifactId=axisg -Dversion=2.2.0 -Dpackaging=jar
+# END of Globus dependencies
+
+#SSH Tools dependencies
+mvn install:install-file -Dfile="lib/gsi-sshterm/TransferAPIClient.jar" -DgroupId=org.globusonline -DartifactId=TransferAPIClient -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/SSHVnc.jar" -DgroupId=com.sshtools.sshvnc -DartifactId=SSHVnc -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/SSHTerm-1.0.0.jar" -DgroupId=com.sshtools.sshterm -DartifactId=SSHTerm -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/ShiFT.jar" -DgroupId=com.sshtools.shift -DartifactId=ShiFT -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/putty-pk-1.1.0.jar" -DgroupId=com.sshtools.ext -DartifactId=putty-pk -Dversion=1.1.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/SecureTunneling.jar" -DgroupId=com.sshtools.tunnel -DartifactId=SecureTunneling -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/not-yet-commons-ssl-0.3.11.jar" -DgroupId=org.apache.commons -DartifactId=not-yet-commons-ssl -Dversion=0.3.11 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/ncsa-lcrypto-146.jar" -DgroupId=edu.illinois.ncsa -DartifactId=ncsa-lcrypto -Dversion=1.4.6 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/libbrowser.jar" -DgroupId=uk.ac.rl.esc.browser -DartifactId=libbrowser -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/jlirc-unix-soc.jar" -DgroupId=org.lirc.socket -DartifactId=jlirc-unix-soc -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/j2ssh-core-0.2.7.jar" -DgroupId=com.sshtools.core -DartifactId=j2ssh-core -Dversion=0.2.7 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/j2ssh-common-0.2.7.jar" -DgroupId=com.sshtools.common -DartifactId=j2ssh-common -Dversion=0.2.7 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/filedrop.jar" -DgroupId=net.iharder.dnd -DartifactId=filedrop -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/commons-compress-1.2.jar" -DgroupId=org.apache.commons -DartifactId=commons-compress -Dversion=1.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/bcprov-jdk15on-1.50.jar" -DgroupId=org.bouncycastle -DartifactId=bcprov -Dversion=1.50.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/BCGSS.jar" -DgroupId=edu.illinois.ncsa -DartifactId=BCGSS -Dversion=1.0.0 -Dpackaging=jar
+mvn install:install-file -Dfile="lib/gsi-sshterm/swing-layout-1.0.3.jar" -DgroupId=org.jdesktop -DartifactId=swing-layout -Dversion=1.0.3 -Dpackaging=jar
+#mvn install:install-file -Dfile="bbbb" -DgroupId=aaaaa -DartifactId=BCGSS -Dversion=1.0.0 -Dpackaging=jar
+
+mvn install:install-file -Dfile="lib/opentsdb/opentsdb-client-2.1.0.jar" -DgroupId=org.opentsdb -DartifactId=opentsdb-client -Dversion=2.1.0 -Dpackaging=jar
+
+echo "FINISHED INSTALLING LOCAL DEPENDENCIES"

build-all.sh

@@ -0,0 +1,23 @@
+if command -v yum > /dev/null 2>&1; then
+    OS="Centos"
+    CMD=yum
+elif command -v zypper > /dev/null 2>&1; then
+    OS="OpenSuse"
+    CMD=zypper
+else
+    OS="Debian"
+    CMD=apt-get
+fi
+if command -v mvn > /dev/null 2>&1; then
+    echo "Maven already installed "
+    echo "Detected OS $OS, using command $CMD"
+else
+    echo "Maven will be installed "
+    sudo $CMD install maven
+fi
+echo "Addling libraries to local maven repository"
+./add-lib-to-local-maven.sh
+
+
+mvn clean install
+

docs/_config.yml

@@ -0,0 +1,7 @@
+title: Fast Data Transfer
+description: FDT is an Application for Efficient Data Transfers which is capable of reading and writing at disk speed over wide area networks (with standard TCP). It is written in Java, runs an all major platforms and it is easy to use.
+show_downloads: true
+google_analytics:
+theme: jekyll-theme-cayman
+gems:
+  - jekyll-menus

docs/doc-examples.md

@@ -0,0 +1,180 @@
+[[Home](index.md)]  [Documentation]  [[Performance Tests](perf-disk-to-disk.md)]
+
+[[FDT & DDCopy](doc-fdt-ddcopy.md)]   [Examples]   [[Security](doc-security.md)]    [[User's Extensions](doc-user-extensions.md)]    [[System Tuning](doc-system-tuning.md)]    [[FDT Monitoring](doc-opentsdb.md)]
+
+### Examples
+
+1. To send one file called "local.data" from the local system
+directory to another computer
+in the "/home/remoteuser/destiantionDir" folder, with default
+parameters, there are two options:
+
+- Client/Server mode
+
+First,the FDT server needs to be started on the remote system. ( The defaultsettings will be used, which implies the default port, 54321, on boththeclient and the server ). -S is used to disable the standalone mode,which means that the server will stop after the session will finish
+
+```
+[remote computer]$ java -jar fdt.jar -S
+```
+
+Then,the client will be started on the local system specifying the sourcefile, the remote address (or hostname) where the server was started inthe previous step and the destination directory
+
+```
+[local computer]$ java -jar fdt.jar -c <remote_address> -d /home/remoteuser/destinationDir /home/localuser/local.data
+```
+
+OR
+
+```
+[local computer]$ java -jar fdt.jar -c <remote_address> -d destinationDir ./local.data
+```
+
+- Secure Copy (SCP) mode
+
+In this mode the server will be started on the remote systemautomatically by the local FDT client using SSH.
+
+```
+[local computer]$ java -jar fdt.jar /home/localuser/local.data remoteuser@<remote_address>:/home/remoteuser/destinationDir
+```
+
+OR
+
+```
+[local computer]$ java -jar fdt.jar ./local.data remoteuser@<remote_address>:destinationDir
+```
+
+If the remoteuser parameter is not specified the local user, runningthe fdt command, will beused to login on the remote system
+
+2. To get the content of an entire folder and all its children,
+located in the user's home directory, the -r ( recursive
+mode ) flag will be specified and also -pull to sink the data from the
+server. In the Client/Server mode the access to the server will be
+restricted to the local IP addresses only ( with -f flag ).
+
+- Client/Server mode
+
+Multiple addresses may be specfied using the -f flag using ':'. If theclient's IP address(es) is not specified in the allowed IP addressesthe connection will be closed. In the following command the server isstarted in standalone mode, which means that will continue to run afterthe session will finish. The transfer rate for every client sessionswill be limited to 4 MBytes/s
+
+```
+[remote computer]$ java -jar fdt.jar -f allowedIP1:allowedIP2 -limit 4M
+```
+
+OR
+
+```
+[remote computer]$ java -jar fdt.jar -f allowedIP1:allowedIP2 -limit 4096K
+```
+
+The command for the local client will be.
+
+```
+[local computer]$ java -jar fdt.jar -pull -r -c <remote_address>-d /home/localuser/localDir /home/remoteuser/remoteDir
+```
+
+OR
+
+```
+[local computer]$ java -jar fdt.jar -pull -r -c <remote_address> -d localDir remoteDir
+```
+
+- SCP mode
+
+In this mode only the order of the parameters will be changed, and -ris the only argument that must be added ( -pull is implicit ). Sameauthentication policies apply as in the first example
+
+```
+[local computer]$ java -jar fdt.jar -r  remoteuser@<remote_address>:/home/remoteuser/remoteDir /home/localuser/localDir
+```
+
+OR
+
+```
+[local computer]$ java -jar fdt.jar -r remoteuser@<remote_address>:remoteDir localDir
+```
+
+3. To test the network connectivity a transfer here is an example
+which transfers data from /dev/zero to /dev/null using 10 streams in
+blocking mode, for both the server and the client with 8 MBytes
+buffers. The server will stop after the test is finished
+
+- Client/Server mode
+
+```
+[remote computer]$ java -jar fdt.jar -bio -bs 8M -f allowedIP -S
+```
+
+```
+[local computer]$ java -jar fdt.jar -c <remote_address> -bio -P 10 -d /dev/null /dev/zero
+```
+
+- SCP mode
+
+```
+[local computer]$ java -jar fdt.jar -bio -P 10 /dev/zero remoteAddress:/dev/null
+```
+
+4. The user can also define a list of files ( a filename per line )
+to be transfered. FDT will detect if the files are located on multiple
+devices and will use a dedicated thread for each device.
+
+```
+[remote computer]$ java -jar fdt.jar -S
+```
+
+```
+[local computer]$ java -jar fdt.jar -fl ./file_list.txt -c <remote_address> -d /home/remoteuser/destDir
+```
+
+5. To test the local read/write performance of the local disk the
+DDCopy may be used.
+
+- The following command will copy the entire partition
+/dev/dsk/c0d1p1 to /dev/null reporting every 2 seconds ( the default )
+the I/O speed
+
+```
+[local computer]$ java -cp fdt.jar lia.util.net.common.DDCopy if=/dev/dsk/c0d1p1 of=/dev/null
+```
+
+- To test the write speed of the file system using a 1GB file
+read from /dev/zero the following command may be used. The operating
+system will sync() the data to the disk. The data will be read/write
+using 10MB buffers
+
+```
+[local computer]$ java -cp fdt.jar lia.util.net.common.DDCopy  if=/dev/zero of=/home/user/1GBTestFile bs=10M count=100 flags=NOSYNC
+```
+
+OR
+
+```
+[local computer]$ java -cp fdt.jar lia.util.net.common.DDCopy  if=/dev/zero of=/home/user/1GBTestFile bs=1M bn=10 count=100 flags=NOSYNC
+```
+
+- Launching FDT as Agent example:
+
+```
+java -jar fdt.jar -tp <transfer,ports,separated,by,comma> -p <portNo> -agent
+```
+
+- Sending coordinator message to the agent:
+
+```
+java -jar fdt.jar -dIP <destination-ip> -dp <destination-port> -sIP <source-ip> -p <source-port> -d /tmp/destination/files -fl /tmp/file-list-on-source.txt -coord
+```
+- Retrieving session log file. 
+
+To retrieve session log file user needs to provide at least these parameters:
+
+```
+java -jar fdt.jar  -c <source-host> -d /tmp/destination/files -sID <session-ID>
+```
+
+- To retrieve list of files on custom path there is a custom mode which can be used.
+
+```
+java -jar fdt.jar  -c <source-host> -ls /tmp/
+```
+
+
+
+