diff --git a/.gradle/7.3.1/checksums/checksums.lock b/.gradle/7.3.1/checksums/checksums.lock
new file mode 100644
index 0000000000000000000000000000000000000000..caab406f36f72d94bc51028636f334712ebf780e
Binary files /dev/null and b/.gradle/7.3.1/checksums/checksums.lock differ
diff --git a/.gradle/7.3.1/checksums/md5-checksums.bin b/.gradle/7.3.1/checksums/md5-checksums.bin
new file mode 100644
index 0000000000000000000000000000000000000000..a4a5a8fdf7f2bd6fd280e1dacc002f40fb747731
Binary files /dev/null and b/.gradle/7.3.1/checksums/md5-checksums.bin differ
diff --git a/.gradle/7.3.1/checksums/sha1-checksums.bin b/.gradle/7.3.1/checksums/sha1-checksums.bin
new file mode 100644
index 0000000000000000000000000000000000000000..0a804dff9a795e99fc677a0dafabaf2b2e77244b
Binary files /dev/null and b/.gradle/7.3.1/checksums/sha1-checksums.bin differ
diff --git a/.gradle/7.3.1/dependencies-accessors/dependencies-accessors.lock b/.gradle/7.3.1/dependencies-accessors/dependencies-accessors.lock
new file mode 100644
index 0000000000000000000000000000000000000000..4277ef310f1e7f84378e9b117ea8a3fa1976108b
Binary files /dev/null and b/.gradle/7.3.1/dependencies-accessors/dependencies-accessors.lock differ
diff --git a/.gradle/7.3.1/dependencies-accessors/gc.properties b/.gradle/7.3.1/dependencies-accessors/gc.properties
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.gradle/7.3.1/executionHistory/executionHistory.bin b/.gradle/7.3.1/executionHistory/executionHistory.bin
new file mode 100644
index 0000000000000000000000000000000000000000..ca9cc04858ff8fcbf97a798b08dd35013aa17e49
Binary files /dev/null and b/.gradle/7.3.1/executionHistory/executionHistory.bin differ
diff --git a/.gradle/7.3.1/executionHistory/executionHistory.lock b/.gradle/7.3.1/executionHistory/executionHistory.lock
new file mode 100644
index 0000000000000000000000000000000000000000..62626269690976d8c429337dbfd60794f36230ac
Binary files /dev/null and b/.gradle/7.3.1/executionHistory/executionHistory.lock differ
diff --git a/.gradle/7.3.1/fileChanges/last-build.bin b/.gradle/7.3.1/fileChanges/last-build.bin
new file mode 100644
index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d
Binary files /dev/null and b/.gradle/7.3.1/fileChanges/last-build.bin differ
diff --git a/.gradle/7.3.1/fileHashes/fileHashes.bin b/.gradle/7.3.1/fileHashes/fileHashes.bin
new file mode 100644
index 0000000000000000000000000000000000000000..2596d1029e15cfc400df7bf623ff24c1c89580b3
Binary files /dev/null and b/.gradle/7.3.1/fileHashes/fileHashes.bin differ
diff --git a/.gradle/7.3.1/fileHashes/fileHashes.lock b/.gradle/7.3.1/fileHashes/fileHashes.lock
new file mode 100644
index 0000000000000000000000000000000000000000..5dccde3dbebe308ce56410d1e7b138966365f31e
Binary files /dev/null and b/.gradle/7.3.1/fileHashes/fileHashes.lock differ
diff --git a/.gradle/7.3.1/fileHashes/resourceHashesCache.bin b/.gradle/7.3.1/fileHashes/resourceHashesCache.bin
new file mode 100644
index 0000000000000000000000000000000000000000..6aafefb919148f3bd074a914aa2fd3e51d09ed6d
Binary files /dev/null and b/.gradle/7.3.1/fileHashes/resourceHashesCache.bin differ
diff --git a/.gradle/7.3.1/gc.properties b/.gradle/7.3.1/gc.properties
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.gradle/buildOutputCleanup/buildOutputCleanup.lock b/.gradle/buildOutputCleanup/buildOutputCleanup.lock
new file mode 100644
index 0000000000000000000000000000000000000000..328268bea1a5f6485d8a4976fa6b4c01acc0eb07
Binary files /dev/null and b/.gradle/buildOutputCleanup/buildOutputCleanup.lock differ
diff --git a/.gradle/buildOutputCleanup/cache.properties b/.gradle/buildOutputCleanup/cache.properties
new file mode 100644
index 0000000000000000000000000000000000000000..8b5c448f7c8b2cffe0a3cf9ad50e3ae98a32d980
--- /dev/null
+++ b/.gradle/buildOutputCleanup/cache.properties
@@ -0,0 +1,2 @@
+#Thu Dec 16 08:27:41 CET 2021
+gradle.version=7.3.1
diff --git a/.gradle/buildOutputCleanup/outputFiles.bin b/.gradle/buildOutputCleanup/outputFiles.bin
new file mode 100644
index 0000000000000000000000000000000000000000..d9782fe1238d6cf62ec632bb11a15d0999884d4a
Binary files /dev/null and b/.gradle/buildOutputCleanup/outputFiles.bin differ
diff --git a/.gradle/checksums/checksums.lock b/.gradle/checksums/checksums.lock
new file mode 100644
index 0000000000000000000000000000000000000000..cd261d8856874b3c4c1ae3265705c627c55453e6
Binary files /dev/null and b/.gradle/checksums/checksums.lock differ
diff --git a/.gradle/checksums/md5-checksums.bin b/.gradle/checksums/md5-checksums.bin
new file mode 100644
index 0000000000000000000000000000000000000000..bbc076898d4ce1bdb6b95fe5ae5cef063fc6d9a9
Binary files /dev/null and b/.gradle/checksums/md5-checksums.bin differ
diff --git a/.gradle/checksums/sha1-checksums.bin b/.gradle/checksums/sha1-checksums.bin
new file mode 100644
index 0000000000000000000000000000000000000000..72b3370f6173b0dfede15133745d5b184ec15275
Binary files /dev/null and b/.gradle/checksums/sha1-checksums.bin differ
diff --git a/.gradle/configuration-cache/gc.properties b/.gradle/configuration-cache/gc.properties
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.gradle/file-system.probe b/.gradle/file-system.probe
new file mode 100644
index 0000000000000000000000000000000000000000..4c679832dbde356fe1d7a0c447f8cc17abdb2008
Binary files /dev/null and b/.gradle/file-system.probe differ
diff --git a/.gradle/vcs-1/gc.properties b/.gradle/vcs-1/gc.properties
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/build.gradle b/build.gradle
index 62c1792da5addd8c0cfb5bd34119db9e8cb597b5..7ab5e9fa60c38e7bf6fccb54329c7d8b42ee46bf 100644
--- a/build.gradle
+++ b/build.gradle
@@ -11,9 +11,7 @@ buildscript {
mavenCentral()
mavenLocal()
maven { url 'https://repo.gradle.org/gradle/libs-releases' }
- jcenter()
maven { url "https://plugins.gradle.org/m2/" }
-
}
dependencies {
// Jar
@@ -51,7 +49,6 @@ jar {
repositories {
// Use jcenter for resolving your dependencies.
// You can declare any Maven/Ivy/file repository here.
- jcenter()
mavenLocal()
}
@@ -64,29 +61,54 @@ version = '0.0.1'
description = """IRODS Retriever and Sender"""
-dependencies {
- // This dependency is found on compile classpath of this component and consumers.
-// compile 'com.google.guava:guava:23.0'
+test {
+ // set heap size for the test JVM(s)
+ minHeapSize = "1280m"
+ maxHeapSize = "15120m"
+
+ // listen to events in the test execution lifecycle
+ beforeTest { descriptor ->
+ logger.lifecycle("Running test: " + descriptor)
+ }
+}
+// Function to download and include URL jar files, now in testing versus using a bash script to prepare the files (still included)
+def urlFile = { url, name ->
+ File file = new File("$buildDir/download/${name}.jar")
+ file.parentFile.mkdirs()
+ if (!file.exists()) {
+ new URL(url).withInputStream { downloadStream ->
+ file.withOutputStream { fileOut ->
+ fileOut << downloadStream
+ }
+ }
+ }
+ files(file.absolutePath)
+}
+
+dependencies {
// Use JUnit test framework
- testCompile 'junit:junit:4.12'
+ testImplementation 'junit:junit:4.13.2'
// Command argument parser
- compile "com.beust:jcommander:1.71"
+ implementation 'com.beust:jcommander:1.81'
- // locally installed jargon and unlock api
- compile group: 'jargon', name: 'core', version: '4.3.0.2'
-// compile group: 'nl.wur.ssb', name: 'unlockapi', version: '1.0.0'
+ // irods dependency obtained locally
+ implementation urlFile('https://github.com/DICE-UNC/jargon/releases/download/4.3.0.2-RELEASE/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar', 'jargon-core-4.3.0.2-RELEASE-jar-with-dependencies')
// https://mvnrepository.com/artifact/commons-validator/commons-validator
- compile group: 'commons-validator', name: 'commons-validator', version: '1.6'
+ implementation group: 'commons-validator', name: 'commons-validator', version: '1.6'
+ // https://mvnrepository.com/artifact/org.slf4j/slf4j-api
+ implementation 'org.apache.logging.log4j:log4j-api:2.16.0'
+ implementation 'org.apache.logging.log4j:log4j-core:2.16.0'
+ implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.16.0'
}
// In this section you declare where to find the dependencies of your project
repositories {
+ mavenCentral()
// Use jcenter for resolving your dependencies.
// You can declare any Maven/Ivy/file repository here.
- jcenter()
}
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
new file mode 100644
index 0000000000000000000000000000000000000000..7454180f2ae8848c63b8b4dea2cb829da983f2fa
Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 0000000000000000000000000000000000000000..84d1f85fd658134f7e982ae908fd537188f20bfe
--- /dev/null
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.1-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
new file mode 100755
index 0000000000000000000000000000000000000000..3da45c161b0b8e84c1a88711f3d98c3b22a28c42
--- /dev/null
+++ b/gradlew
@@ -0,0 +1,234 @@
+#!/bin/sh
+
+#
+# Copyright ? 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+#
+# Gradle start up script for POSIX generated by Gradle.
+#
+# Important for running:
+#
+# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+# noncompliant, but you have some other compliant shell such as ksh or
+# bash, then to run this script, type that shell name before the whole
+# command line, like:
+#
+# ksh Gradle
+#
+# Busybox and similar reduced shells will NOT work, because this script
+# requires all of these POSIX shell features:
+# * functions;
+# * expansions ?$var?, ?${var}?, ?${var:-default}?, ?${var+SET}?,
+# ?${var#prefix}?, ?${var%suffix}?, and ?$( cmd )?;
+# * compound commands having a testable exit status, especially ?case?;
+# * various built-in commands including ?command?, ?set?, and ?ulimit?.
+#
+# Important for patching:
+#
+# (2) This script targets any POSIX shell, so it avoids extensions provided
+# by Bash, Ksh, etc; in particular arrays are avoided.
+#
+# The "traditional" practice of packing multiple parameters into a
+# space-separated string is a well documented source of bugs and security
+# problems, so this is (mostly) avoided, by progressively accumulating
+# options in "$@", and eventually passing that to Java.
+#
+# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+# see the in-line comments for details.
+#
+# There are tweaks for specific operating systems such as AIX, CygWin,
+# Darwin, MinGW, and NonStop.
+#
+# (3) This script is generated from the Groovy template
+# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+# within the Gradle project.
+#
+# You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+ APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
+ [ -h "$app_path" ]
+do
+ ls=$( ls -ld "$app_path" )
+ link=${ls#*' -> '}
+ case $link in #(
+ /*) app_path=$link ;; #(
+ *) app_path=$APP_HOME$link ;;
+ esac
+done
+
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+
+APP_NAME="Gradle"
+APP_BASE_NAME=${0##*/}
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+ echo "$*"
+} >&2
+
+die () {
+ echo
+ echo "$*"
+ echo
+ exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in #(
+ CYGWIN* ) cygwin=true ;; #(
+ Darwin* ) darwin=true ;; #(
+ MSYS* | MINGW* ) msys=true ;; #(
+ NONSTOP* ) nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD=$JAVA_HOME/jre/sh/java
+ else
+ JAVACMD=$JAVA_HOME/bin/java
+ fi
+ if [ ! -x "$JAVACMD" ] ; then
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+else
+ JAVACMD=java
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+ case $MAX_FD in #(
+ max*)
+ MAX_FD=$( ulimit -H -n ) ||
+ warn "Could not query maximum file descriptor limit"
+ esac
+ case $MAX_FD in #(
+ '' | soft) :;; #(
+ *)
+ ulimit -n "$MAX_FD" ||
+ warn "Could not set maximum file descriptor limit to $MAX_FD"
+ esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+# * args from the command line
+# * the main class name
+# * -classpath
+# * -D...appname settings
+# * --module-path (only if needed)
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+ APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+ CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+ JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
+ for arg do
+ if
+ case $arg in #(
+ -*) false ;; # don't mess with options #(
+ /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
+ [ -e "$t" ] ;; #(
+ *) false ;;
+ esac
+ then
+ arg=$( cygpath --path --ignore --mixed "$arg" )
+ fi
+ # Roll the args list around exactly as many times as the number of
+ # args, so each arg winds up back in the position where it started, but
+ # possibly modified.
+ #
+ # NB: a `for` loop captures its iteration list before it begins, so
+ # changing the positional parameters here affects neither the number of
+ # iterations, nor the values presented in `arg`.
+ shift # remove old arg
+ set -- "$@" "$arg" # push replacement arg
+ done
+fi
+
+# Collect all arguments for the java command;
+# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+# shell script including quotes and variable substitutions, so put them in
+# double quotes to make sure that they get re-expanded; and
+# * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+ "-Dorg.gradle.appname=$APP_BASE_NAME" \
+ -classpath "$CLASSPATH" \
+ org.gradle.wrapper.GradleWrapperMain \
+ "$@"
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+# set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+ printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+ xargs -n1 |
+ sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+ tr '\n' ' '
+ )" '"$@"'
+
+exec "$JAVACMD" "$@"
diff --git a/gradlew.bat b/gradlew.bat
new file mode 100644
index 0000000000000000000000000000000000000000..ac1b06f93825db68fb0c0b5150917f340eaa5d02
--- /dev/null
+++ b/gradlew.bat
@@ -0,0 +1,89 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega
diff --git a/install.sh b/install.sh
index 362aca7cd700899011bd8e86358dec19aebf642e..cc83371ab2e01522969571afda0e52877b2f7126 100755
--- a/install.sh
+++ b/install.sh
@@ -11,10 +11,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
git -C $DIR pull
-# Jargon dependency
-wget -nc https://github.com/DICE-UNC/jargon/releases/download/4.3.0.2-RELEASE/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar -O $DIR/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar
-mvn install:install-file -Dfile=$DIR/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar -DgroupId=jargon -DartifactId=core -Dversion=4.3.0.2 -Dpackaging=jar
-
# Building mode
if [ "$1" == "test" ]; then
gradle build -b "$DIR/build.gradle" --info
diff --git a/src/main/java/nl/wur/ssb/Download.java b/src/main/java/nl/wur/ssb/Download.java
index 0d5ccec3c8bda52af4bba8ebeea91a850ca13e78..b87a1992a11a7d3db48ea77627fb516a7673d6e7 100644
--- a/src/main/java/nl/wur/ssb/Download.java
+++ b/src/main/java/nl/wur/ssb/Download.java
@@ -1,23 +1,37 @@
package nl.wur.ssb;
+import org.apache.commons.lang3.StringUtils;
import org.irods.jargon.core.checksum.ChecksumValue;
import org.irods.jargon.core.checksum.LocalChecksumComputerFactory;
import org.irods.jargon.core.checksum.LocalChecksumComputerFactoryImpl;
import org.irods.jargon.core.checksum.SHA256LocalChecksumComputerStrategy;
+import org.irods.jargon.core.connection.ConnectionProgressStatusListener;
import org.irods.jargon.core.exception.JargonException;
+import org.irods.jargon.core.packinstr.TransferOptions;
import org.irods.jargon.core.protovalues.ChecksumEncodingEnum;
import org.irods.jargon.core.pub.DataObjectChecksumUtilitiesAO;
import org.irods.jargon.core.pub.DataTransferOperations;
+import org.irods.jargon.core.pub.DefaultIntraFileProgressCallbackListener;
import org.irods.jargon.core.pub.io.IRODSFile;
import org.irods.jargon.core.query.GenQueryBuilderException;
import org.irods.jargon.core.query.JargonQueryException;
+import org.irods.jargon.core.transfer.DefaultTransferControlBlock;
+import org.irods.jargon.core.transfer.TransferControlBlock;
+import org.irods.jargon.core.transfer.TransferStatus;
+import org.irods.jargon.core.transfer.TransferStatusCallbackListener;
+import org.irods.jargon.core.utils.Host;
import java.io.File;
import java.io.FileNotFoundException;
import java.nio.file.FileAlreadyExistsException;
import java.util.HashSet;
+import java.util.logging.Level;
import java.util.logging.Logger;
+import static java.util.logging.Level.INFO;
+import static org.apache.log4j.Level.*;
+import static org.irods.jargon.core.utils.IRODSConstants.status;
+
public class Download {
private static final java.util.logging.Logger logger = Logger.getLogger(Upload.class.getName());
@@ -99,23 +113,35 @@ public class Download {
if (localFile.exists()) {
if (localFile.isFile()) {
if (commandOptions.force) {
- // Get local HASH
- LocalChecksumComputerFactory factory = new LocalChecksumComputerFactoryImpl();
- SHA256LocalChecksumComputerStrategy localActual = (SHA256LocalChecksumComputerStrategy) factory.instance(ChecksumEncodingEnum.SHA256);
- ChecksumValue localChecksumValue = localActual.computeChecksumValueForLocalFile(localFile.getAbsolutePath());
- // Get remote hash
- DataObjectChecksumUtilitiesAO dataObjectChecksumUtilitiesAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataObjectChecksumUtilitiesAO(connection.irodsAccount);
- ChecksumValue remoteChecksumValue = dataObjectChecksumUtilitiesAO.computeChecksumOnDataObject(irodsFile);
- if (localChecksumValue.getBase64ChecksumValue().contains(remoteChecksumValue.getBase64ChecksumValue())) {
- logger.info("Same checksum, not going to overwrite");
- return;
+ // If bigger than 50gb... do a byte size check?
+ if (localFile.length() / (1024 * 1024 * 1024) > 50) {
+ if (localFile.length() != irodsFile.length()) {
+ // Delete local file
+ while (localFile.exists()) {
+ localFile.delete();
+ }
+ } else {
+ logger.info("File is big thus skipping checksum. Byte size is the same");
+ }
} else {
- logger.info("Remove local file location " + localFile + " " + localChecksumValue.getBase64ChecksumValue());
- logger.info("Does not match checksum of " + irodsFile.getAbsolutePath() + " " + remoteChecksumValue.getBase64ChecksumValue());
- localFile.delete();
+ // Get local HASH
+ logger.info("Computing local checksum of " + localFile);
+ LocalChecksumComputerFactory factory = new LocalChecksumComputerFactoryImpl();
+ SHA256LocalChecksumComputerStrategy localActual = (SHA256LocalChecksumComputerStrategy) factory.instance(ChecksumEncodingEnum.SHA256);
+ ChecksumValue localChecksumValue = localActual.computeChecksumValueForLocalFile(localFile.getAbsolutePath());
+ // Get remote hash
+ DataObjectChecksumUtilitiesAO dataObjectChecksumUtilitiesAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataObjectChecksumUtilitiesAO(connection.irodsAccount);
+ ChecksumValue remoteChecksumValue = dataObjectChecksumUtilitiesAO.computeChecksumOnDataObject(irodsFile);
+ if (localChecksumValue.getBase64ChecksumValue().contains(remoteChecksumValue.getBase64ChecksumValue())) {
+ logger.info("Same checksum, not going to overwrite");
+ return;
+ } else {
+ logger.info("Remove local file location " + localFile + " " + localChecksumValue.getBase64ChecksumValue());
+ logger.info("Does not match checksum of " + irodsFile.getAbsolutePath() + " " + remoteChecksumValue.getBase64ChecksumValue());
+ localFile.delete();
+ }
}
} else {
- logger.info("force: " + commandOptions.force);
throw new FileAlreadyExistsException("File already exists use --force to overwrite the file: " + localFile);
}
}
@@ -129,11 +155,22 @@ public class Download {
// THE ACTUAL TRANSFER
// Disables the logger for the transfer as it easily gives thousands of lines... and perform the transfer
-// org.apache.log4j.Logger.getLogger("org.irods.jargon.core.transfer").setLevel(Level.OFF);
+ org.apache.log4j.Logger.getLogger("org.irods.jargon.core.transfer").setLevel(org.apache.log4j.Level.INFO);
if (!localFile.exists()) {
logger.info("Downloading file from " + irodsFile + " to " + localFile.getAbsolutePath());
try {
- dataTransferOperationsAO.getOperation(irodsFile, localFile, null, null);
+ StatusCallbackListener statusCallbackListener = new StatusCallbackListener();
+
+ TransferControlBlock defaultTransferControlBlock = DefaultTransferControlBlock.instance();
+ TransferOptions transferOptions = dataTransferOperationsAO.buildTransferOptionsBasedOnJargonProperties();
+ transferOptions.setIntraFileStatusCallbacks(true);
+ transferOptions.setIntraFileStatusCallbacksNumberCallsInterval(1);
+ transferOptions.setIntraFileStatusCallbacksTotalBytesInterval(2);
+ System.err.println("isComputeChecksumAfterTransfer " + transferOptions.isComputeChecksumAfterTransfer());
+ System.err.println("isComputeAndVerifyChecksumAfterTransfer " + transferOptions.isComputeAndVerifyChecksumAfterTransfer());
+ defaultTransferControlBlock.setTransferOptions(transferOptions);
+ dataTransferOperationsAO.getOperation(irodsFile, localFile, statusCallbackListener, defaultTransferControlBlock);
+ logger.info("Transfer completed");
} catch (JargonException e) {
if (localFile.exists()) {
localFile.delete();
@@ -141,18 +178,30 @@ public class Download {
throw new JargonException("Download failed");
}
- // Checksum test!
- LocalChecksumComputerFactory factory = new LocalChecksumComputerFactoryImpl();
- SHA256LocalChecksumComputerStrategy localActual = (SHA256LocalChecksumComputerStrategy) factory.instance(ChecksumEncodingEnum.SHA256);
- ChecksumValue localChecksumValue = localActual.computeChecksumValueForLocalFile(localFile.getAbsolutePath());
- // Get remote hash
- DataObjectChecksumUtilitiesAO dataObjectChecksumUtilitiesAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataObjectChecksumUtilitiesAO(connection.irodsAccount);
- ChecksumValue remoteChecksumValue = dataObjectChecksumUtilitiesAO.computeChecksumOnDataObject(irodsFile);
- if (!localChecksumValue.getBase64ChecksumValue().contains(remoteChecksumValue.getBase64ChecksumValue())) {
- logger.info("Download failed! Incomplete / different checksum detected " + localFile + " " + localChecksumValue.getBase64ChecksumValue());
- logger.info("Does not match checksum of " + irodsFile.getAbsolutePath() + " " + remoteChecksumValue.getBase64ChecksumValue());
- localFile.delete();
- System.exit(2);
+ // Size limit to 50gb for checksum
+ if (localFile.length() / (1024 * 1024 * 1024) > 50) {
+ if (localFile.length() != irodsFile.length()) {
+ // Delete local file
+ while (localFile.exists()) {
+ localFile.delete();
+ }
+ } else {
+ logger.info("File is too large for checksum validation but byte validation passed");
+ }
+ } else {
+ // Checksum test!
+ LocalChecksumComputerFactory factory = new LocalChecksumComputerFactoryImpl();
+ SHA256LocalChecksumComputerStrategy localActual = (SHA256LocalChecksumComputerStrategy) factory.instance(ChecksumEncodingEnum.SHA256);
+ ChecksumValue localChecksumValue = localActual.computeChecksumValueForLocalFile(localFile.getAbsolutePath());
+ // Get remote hash
+ DataObjectChecksumUtilitiesAO dataObjectChecksumUtilitiesAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataObjectChecksumUtilitiesAO(connection.irodsAccount);
+ ChecksumValue remoteChecksumValue = dataObjectChecksumUtilitiesAO.computeChecksumOnDataObject(irodsFile);
+ if (!localChecksumValue.getBase64ChecksumValue().contains(remoteChecksumValue.getBase64ChecksumValue())) {
+ logger.info("Download failed! Incomplete / different checksum detected " + localFile + " " + localChecksumValue.getBase64ChecksumValue());
+ logger.info("Does not match checksum of " + irodsFile.getAbsolutePath() + " " + remoteChecksumValue.getBase64ChecksumValue());
+ localFile.delete();
+ System.exit(2);
+ }
}
}
}
diff --git a/src/main/java/nl/wur/ssb/StatusCallbackListener.java b/src/main/java/nl/wur/ssb/StatusCallbackListener.java
new file mode 100644
index 0000000000000000000000000000000000000000..ab544d2cab6fe28eb724ef3d48a1712f10411280
--- /dev/null
+++ b/src/main/java/nl/wur/ssb/StatusCallbackListener.java
@@ -0,0 +1,126 @@
+package nl.wur.ssb;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.irods.jargon.core.transfer.TransferStatus;
+import org.irods.jargon.core.transfer.TransferStatus.TransferState;
+import org.irods.jargon.core.transfer.TransferStatusCallbackListener;
+
+public class StatusCallbackListener implements TransferStatusCallbackListener {
+
+ private int successCallbackCount = 0;
+ private int errorCallbackCount = 0;
+ private String lastSourcePath = "";
+ private String lastTargetPath = "";
+ private String lastResource = "";
+ private TransferStatusCallbackListener.CallbackResponse forceOption = TransferStatusCallbackListener.CallbackResponse.NO_FOR_ALL;
+
+ private long bytesReportedIntraFileCallbacks = 0L;
+ private int numberIntraFileCallbacks = 0;
+
+ private final List<TransferStatus> errorCallbacks = new ArrayList<>();
+ private long transfered = 0L;
+
+ public List<TransferStatus> getErrorCallbacks() {
+ return errorCallbacks;
+ }
+
+ @Override
+ public FileStatusCallbackResponse statusCallback(final TransferStatus transferStatus) {
+ // Reporting per GB
+ long gbs = Math.round((double) transferStatus.getBytesTransfered() / (1024 * 1024 * 1024));
+ if (transfered < gbs) {
+ long totalSize = transferStatus.getTotalSize() / (1024 * 1024 * 1024);
+ System.err.println("Transfer status: " + transfered + "GB of " + totalSize + "GB " + String.format("%.2f", ((double) transfered / (double) totalSize) * 100.0) + "%");
+ transfered = gbs;
+ }
+ if (transferStatus.isIntraFileStatusReport()) {
+ numberIntraFileCallbacks++;
+ bytesReportedIntraFileCallbacks = transferStatus.getBytesTransfered();
+ } else if (transferStatus.getTransferState() == TransferState.FAILURE) {
+ errorCallbackCount++;
+ } else if (transferStatus.getTransferState() == TransferState.IN_PROGRESS_START_FILE) {
+ // ignored
+ } else {
+ successCallbackCount++;
+ lastSourcePath = transferStatus.getSourceFileAbsolutePath();
+ lastTargetPath = transferStatus.getTargetFileAbsolutePath();
+ lastResource = transferStatus.getTargetResource();
+ }
+
+ if (transferStatus.getTransferException() != null) {
+ errorCallbacks.add(transferStatus);
+ }
+ return FileStatusCallbackResponse.CONTINUE;
+
+ }
+
+ public int getSuccessCallbackCount() {
+ return successCallbackCount;
+ }
+
+ public void setSuccessCallbackCount(final int successCallbackCount) {
+ this.successCallbackCount = successCallbackCount;
+ }
+
+ public int getErrorCallbackCount() {
+ return errorCallbackCount;
+ }
+
+ public void setErrorCallbackCount(final int errorCallbackCount) {
+ this.errorCallbackCount = errorCallbackCount;
+ }
+
+ public String getLastSourcePath() {
+ return lastSourcePath;
+ }
+
+ public String getLastTargetPath() {
+ return lastTargetPath;
+ }
+
+ public String getLastResource() {
+ return lastResource;
+ }
+
+ @Override
+ public void overallStatusCallback(final TransferStatus transferStatus) {
+ if (transferStatus.getTransferState() == TransferState.FAILURE) {
+ errorCallbackCount++;
+ } else {
+ successCallbackCount++;
+ }
+ lastSourcePath = transferStatus.getSourceFileAbsolutePath();
+ lastTargetPath = transferStatus.getTargetFileAbsolutePath();
+ lastResource = transferStatus.getTargetResource();
+ }
+
+ /**
+ * @return the bytesReportedIntraFileCallbacks
+ */
+ public long getBytesReportedIntraFileCallbacks() {
+ return bytesReportedIntraFileCallbacks;
+ }
+
+ /**
+ * @return the numberIntraFileCallbacks
+ */
+ public int getNumberIntraFileCallbacks() {
+ return numberIntraFileCallbacks;
+ }
+
+ @Override
+ public CallbackResponse transferAsksWhetherToForceOperation(final String irodsAbsolutePath,
+ final boolean isCollection) {
+ return forceOption;
+ }
+
+ public CallbackResponse getForceOption() {
+ return forceOption;
+ }
+
+ public void setForceOption(final CallbackResponse forceOption) {
+ this.forceOption = forceOption;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/nl/wur/ssb/Upload.java b/src/main/java/nl/wur/ssb/Upload.java
index 4d9475bb481967154dcc634778c2d2122cb82696..24d619bacad7c76cc7353af15ef0f3b7f335f9a8 100644
--- a/src/main/java/nl/wur/ssb/Upload.java
+++ b/src/main/java/nl/wur/ssb/Upload.java
@@ -37,10 +37,9 @@ public class Upload {
for (File localFile : localFolder.listFiles()) {
if (localFile.isDirectory()) {
- logger.info("Directory detected");
+ logger.debug("Directory detected");
uploadCollectionContent(commandOptions, connection, localFile, targetFolder);
} else {
- logger.info("Uploading " + localFile + " to " + targetFolder + "/" + localFile.getName());
IRODSFile destFile = connection.fileFactory.instanceIRODSFile(targetFolder + "/" + localFile.getName());
if (destFile.exists() && commandOptions.force) {
// Overwrite the file, need to find out how without deleting first
@@ -48,8 +47,9 @@ public class Upload {
}
if (destFile.exists()) {
- logger.warn("Skipping file as it exists " + destFile);
+ logger.debug("Skipping file as it exists " + destFile);
} else {
+ logger.info("Uploading " + localFile + " to " + targetFolder + "/" + localFile.getName());
dataTransferOperationsAO.putOperation(localFile, destFile, null, null);
}
}