!/bin/bash
#
# cndaBulkDownload.sh
#
# Mode 1 (whole project):  -u <user> -p <project> [-t <xsitype>] [-d <dir>]
# Mode 2 (pick sessions):  -u <user> -s <sessionlist.csv> [-d <dir>]
#   CSV format, no header: Project,Subject,Session
#
# Credentials, instead of typing a password every time it's needed:
#   -c / --creds <creds.json>   JSON file: {"user": "username", "password": "..."}
#   Do not use -u together with -c; the username comes from the file.
#   The password is never placed on the command line or in curl's argv --
#   it's written to a short-lived curl config file (chmod 600) and passed
#   via `curl -K`, then deleted on exit. Protect the creds.json file itself
#   (chmod 600) since it holds a plaintext password at rest.
#   No extra tools needed to read the file -- JSON parsing is plain bash.
#
# Requires bash + curl + mktemp. On Windows: Git Bash or WSL.
#
# Env vars:
#   CURL_INSECURE=1     only if TLS verification fails and you understand why (default 0)
#   STRICT_ZIP_CHECK=1  fully validate archive integrity with `unzip -t` (slower, default 0)
#   STALL_TIME=300      abort a transfer averaging below STALL_LIMIT bytes/sec for this many seconds
#   STALL_LIMIT=1024    bytes/sec threshold for the stall check above
#   SESSION_TIMEOUT=1800   how long a CNDA JSESSION lasts, in seconds (default 1800 = 30 min)
#   REFRESH_MARGIN=300     proactively reauth if within this many seconds of SESSION_TIMEOUT (default 300 = 5 min)
#
set -uo pipefail
umask 077
 
WRKDIR=$(pwd -P)
echo "Working directory (logs, temp files, relative paths resolve here): $WRKDIR"
XNATHOST=https://cnda.wustl.edu
XSITYPES=xnat:mrSessionData
CURL_INSECURE=${CURL_INSECURE:-0}
STRICT_ZIP_CHECK=${STRICT_ZIP_CHECK:-0}
STALL_TIME=${STALL_TIME:-300}
STALL_LIMIT=${STALL_LIMIT:-1024}
SESSION_TIMEOUT=${SESSION_TIMEOUT:-1800}
REFRESH_MARGIN=${REFRESH_MARGIN:-300}
 
for cmd in curl mktemp; do
  command -v "$cmd" >/dev/null 2>&1 || { echo "Required command not found: $cmd" >&2; exit 1; }
done
if [ "$STRICT_ZIP_CHECK" = "1" ]; then
  command -v unzip >/dev/null 2>&1 || { echo "STRICT_ZIP_CHECK=1 requires unzip, which was not found." >&2; exit 1; }
fi
 
for _var in CURL_INSECURE STRICT_ZIP_CHECK; do
  if [ "${!_var}" != "0" ] && [ "${!_var}" != "1" ]; then
    echo "$_var must be 0 or 1 (got '${!_var}')" >&2
    exit 2
  fi
done
 
is_positive_int() {
  [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
}
for _var in STALL_TIME STALL_LIMIT SESSION_TIMEOUT REFRESH_MARGIN; do
  if ! is_positive_int "${!_var}"; then
    echo "Invalid value for $_var: '${!_var}' (must be a positive integer)" >&2
    exit 2
  fi
done
if [ "$REFRESH_MARGIN" -ge "$SESSION_TIMEOUT" ]; then
  echo "REFRESH_MARGIN ($REFRESH_MARGIN) must be less than SESSION_TIMEOUT ($SESSION_TIMEOUT) -- as set, every download would trigger a reauth." >&2
  exit 2
fi
unset _var
 
is_safe_component() {
  [[ "$1" =~ ^[A-Za-z0-9._-]+$ ]]
}
 
USAGE() {
  echo "Usage: cndaBulkDownload.sh [-u <user id> | -c <creds.json>] [-p <project> | -s <session list csv>]"
  echo "  Whole project:  -p <project> [-t <xsitype>, default xnat:mrSessionData]"
  echo "  Pick sessions:  -s <csv: Project,Subject,Session per line, no header>"
  echo "  Credentials:    -u <user id>  (prompts for password each time it's needed)"
  echo "               or -c/--creds <creds.json>  {\"user\": \"username\", \"password\": \"...\"}"
  echo "  Optional:       -d <download directory>"
  echo "  Env:            CURL_INSECURE=1, STRICT_ZIP_CHECK=1, STALL_TIME, STALL_LIMIT (see header comment)"
}
 
need_val() { [ $# -ge 2 ] || { echo "Option $1 requires a value." >&2; USAGE; exit 2; }; }
 
while [ $# -gt 0 ]; do
  case "$1" in
    -u) need_val "$@"; USERID=$2; shift 2 ;;
    -p) need_val "$@"; PROJECT=$2; shift 2 ;;
    -t) need_val "$@"; XSITYPES=$2; shift 2 ;;
    -d) need_val "$@"; DOWNLOADDIR=$2; shift 2 ;;
    -s) need_val "$@"; SESSIONLIST=$2; shift 2 ;;
    -c|--creds) need_val "$@"; CREDSFILE=$2; shift 2 ;;
    *) echo "Unknown option: $1" >&2; USAGE; exit 2 ;;
  esac
done
 
if [ -n "${CREDSFILE:-}" ]; then
  if [ -n "${USERID:-}" ]; then
    echo "Use either -u or -c/--creds, not both (username comes from the creds file)." >&2
    exit 2
  fi
  [ -f "$CREDSFILE" ] || { echo "Creds file not found: $CREDSFILE" >&2; exit 2; }
 
  # Best-effort warning if the creds file is readable by group/other. Not fatal --
  # permission semantics differ across Linux/macOS/Git Bash -- but worth flagging.
  CREDS_PERMS=$(stat -c '%a' "$CREDSFILE" 2>/dev/null || stat -f '%Lp' "$CREDSFILE" 2>/dev/null || echo "")
  if [ -n "$CREDS_PERMS" ]; then
    GROUP_OTHER_BITS=${CREDS_PERMS: -2}
    if [ "$GROUP_OTHER_BITS" != "00" ]; then
      echo "Warning: $CREDSFILE may be readable by others (perms: $CREDS_PERMS). Consider: chmod 600 \"$CREDSFILE\"" >&2
    fi
  fi
 
  extract_json_field() {
    # Pure-bash JSON string field extractor. No external tools required.
    # Handles \" \\ \n \t escapes correctly, unlike a plain regex/sed match.
    local file=$1 key=$2
    local content rest after i c nc result len
    content=$(cat "$file")
    rest=${content#*\"$key\"}
    [ "$rest" = "$content" ] && return 1   # key not found
    rest=${rest#*:}
    while [ "${rest:0:1}" = " " ] || [ "${rest:0:1}" = $'\t' ] || [ "${rest:0:1}" = $'\n' ]; do
      rest=${rest:1}
    done
    [ "${rest:0:1}" = '"' ] || return 1
    after=${rest:1}
    result=""
    i=0
    len=${#after}
    while [ "$i" -lt "$len" ]; do
      c="${after:$i:1}"
      if [ "$c" = '\' ]; then
        i=$((i+1))
        nc="${after:$i:1}"
        case "$nc" in
          '"') result+='"' ;;
          '\') result+='\' ;;
          n) result+=$'\n' ;;
          t) result+=$'\t' ;;
          *) result+="$nc" ;;
        esac
      elif [ "$c" = '"' ]; then
        break
      else
        result+="$c"
      fi
      i=$((i+1))
    done
    printf '%s' "$result"
  }
 
  USERID=$(extract_json_field "$CREDSFILE" user)
  CREDS_PASSWORD=$(extract_json_field "$CREDSFILE" password)
 
  if [ -z "$USERID" ] || [ -z "$CREDS_PASSWORD" ]; then
    echo "Could not read 'user' and 'password' fields from $CREDSFILE" >&2
    echo "Expected format: {\"user\": \"username\", \"password\": \"...\"}" >&2
    exit 2
  fi
 
  # Reject embedded newlines/carriage returns -- the curl config file is
  # line-oriented, so a control character in either field could inject an
  # extra directive rather than just being part of the value.
  case "$USERID" in *$'\n'*|*$'\r'*)
    echo "Invalid 'user' value in $CREDSFILE: contains a newline or carriage return." >&2
    exit 2 ;;
  esac
  case "$CREDS_PASSWORD" in *$'\n'*|*$'\r'*)
    echo "Invalid 'password' value in $CREDSFILE: contains a newline or carriage return." >&2
    exit 2 ;;
  esac
fi
 
[ -z "${USERID:-}" ] && { echo "No user id entered (use -u or -c/--creds)" >&2; USAGE; exit 2; }
if [ -z "${PROJECT:-}" ] && [ -z "${SESSIONLIST:-}" ]; then
  echo "Specify -p <project> or -s <csv>" >&2; USAGE; exit 2
fi
if [ -n "${PROJECT:-}" ] && [ -n "${SESSIONLIST:-}" ]; then
  echo "Use either -p or -s, not both." >&2; exit 2
fi
if [ -n "${PROJECT:-}" ] && ! is_safe_component "$PROJECT"; then
  echo "Invalid or unsafe project identifier: $PROJECT" >&2
  exit 2
fi
if [ -n "${SESSIONLIST:-}" ]; then
  [ -f "$SESSIONLIST" ] || { echo "Session list not found: $SESSIONLIST" >&2; exit 2; }
  SESSIONLIST=$(cd "$(dirname "$SESSIONLIST")" && pwd -P)/$(basename "$SESSIONLIST")
fi
 
DOWNLOADDIR=${DOWNLOADDIR:-"${PROJECT:-cnda}_downloads"}
if ! mkdir -p "$DOWNLOADDIR"; then
  echo "Could not create download directory: $DOWNLOADDIR" >&2; exit 1
fi
if ! cd "$DOWNLOADDIR"; then
  echo "Could not enter download directory: $DOWNLOADDIR" >&2; exit 1
fi
 
LOGDIR="$WRKDIR/logs"
if ! mkdir -p "$LOGDIR"; then
  echo "Could not create log directory: $LOGDIR" >&2; exit 1
fi
LOGFILE="$LOGDIR/cndaBulkDownload_$(date '+%Y%m%d_%H%M%S').log"
: > "$LOGFILE"
 
CURLOPTS=(
  -sS
  --fail
  --location
  --connect-timeout 20
  --speed-time "$STALL_TIME"
  --speed-limit "$STALL_LIMIT"
  --retry 2
  --retry-delay 5
)
[ "$CURL_INSECURE" = "1" ] && CURLOPTS+=(-k)
 
CFGFILE=$(mktemp "$WRKDIR/.RESTresults.XXXXXX") || {
  echo "Could not create temporary file in $WRKDIR" >&2
  exit 1
}
 
OK_COUNT=0
FAIL_COUNT=0
SKIP_COUNT=0
 
log() { echo "$(date '+%F %T') $1" >> "$LOGFILE"; }
 
# If a creds file was supplied, write a short-lived curl config file holding
# the credentials, so the password never appears on the command line or in
# `ps` output. umask 077 (set at script start) means mktemp already creates
# this as 600, but we chmod explicitly to be sure regardless of platform.
CURLCFG=""
if [ -n "${CREDS_PASSWORD:-}" ]; then
  escape_curlcfg() {
    local v=$1
    v=${v//\\/\\\\}
    v=${v//\"/\\\"}
    printf '%s' "$v"
  }
  CURLCFG=$(mktemp "$WRKDIR/.curlcfg.XXXXXX") || {
    echo "Could not create temporary curl config file in $WRKDIR" >&2
    exit 1
  }
  chmod 600 "$CURLCFG"
  printf 'user = "%s:%s"\n' "$(escape_curlcfg "$USERID")" "$(escape_curlcfg "$CREDS_PASSWORD")" > "$CURLCFG"
  CREDS_PASSWORD=  # no longer needed in the shell variable once written to the config file
fi
 
logout_session() {
  if [ -n "${JSESSION:-}" ]; then
    curl "${CURLOPTS[@]}" -X DELETE -b "JSESSIONID=$JSESSION" "$XNATHOST/REST/JSESSION/" >/dev/null 2>&1 || true
    JSESSION=
  fi
}
 
cleanup() {
  logout_session
  rm -f "$CFGFILE"
  [ -n "$CURLCFG" ] && rm -f "$CURLCFG"
}
trap cleanup EXIT
 
authenticate() {
  logout_session
 
  if [ -n "$CURLCFG" ]; then
    JSESSION=$(curl "${CURLOPTS[@]}" -K "$CURLCFG" "$XNATHOST/REST/JSESSION")
    auth_rc=$?
  else
    JSESSION=$(curl "${CURLOPTS[@]}" -u "$USERID" "$XNATHOST/REST/JSESSION")
    auth_rc=$?
  fi
 
  if [ "$auth_rc" -ne 0 ]; then
    echo "Unable to contact CNDA authentication endpoint." >&2
    log "AUTH_REQUEST_FAIL"
    exit 1
  fi
  JSESSION=${JSESSION//$'\r'/}
  JSESSION=${JSESSION//$'\n'/}
  if [ -z "$JSESSION" ] || ! [[ "$JSESSION" =~ ^[A-Za-z0-9._-]+$ ]]; then
    echo "CNDA login failed. Check your credentials and retry." >&2
    log "AUTH_FAIL"
    exit 1
  fi
  LAST_AUTH_TIME=$(date +%s)
}
 
# CNDA JSESSIONs expire after SESSION_TIMEOUT seconds. Rather than waiting
# for a download to fail and retrying reactively, check the age of the
# current session before each download and refresh proactively if we're
# within REFRESH_MARGIN seconds of the expected expiry. This avoids paying
# for a failed attempt + reauth + repeat on every session once the token
# goes stale during a long run.
refresh_session_if_stale() {
  local now elapsed
  now=$(date +%s)
  elapsed=$((now - LAST_AUTH_TIME))
  if [ "$elapsed" -ge "$((SESSION_TIMEOUT - REFRESH_MARGIN))" ]; then
    authenticate
  fi
}
 
is_valid_zip() {
  [ -s "$1" ] || return 1
  [ "$(head -c 2 "$1")" = "PK" ] || return 1
  if [ "$STRICT_ZIP_CHECK" = "1" ]; then
    unzip -tqq "$1" >/dev/null 2>&1 || return 1
  fi
  return 0
}
 
clean_field() {
  local v=$1
  v=${v//$'\r'/}
  v=${v#\"}; v=${v%\"}
  printf '%s' "$v"
}
 
download_session() {
  local project=$1 subj=$2 label=$3
 
  refresh_session_if_stale
 
  if ! is_safe_component "$project" || ! is_safe_component "$subj" || ! is_safe_component "$label"; then
    echo "  -> skipping unsafe identifier: project=$project subject=$subj session=$label" >&2
    log "INVALID_IDENTIFIER project=$project subject=$subj session=$label"
    FAIL_COUNT=$((FAIL_COUNT+1))
    return
  fi
 
  local dir="$project/$subj"
  local zipfile="$dir/${label}.zip"
  local tmpfile="${zipfile}.part"
 
  if ! mkdir -p "$dir"; then
    echo "  -> FAILED: could not create directory $dir" >&2
    log "FAIL_CREATE_DIR $project/$subj/$label"
    FAIL_COUNT=$((FAIL_COUNT+1))
    return
  fi
 
  if is_valid_zip "$zipfile"; then
    echo "$zipfile already exists, skipping"
    log "SKIP $project/$subj/$label"
    SKIP_COUNT=$((SKIP_COUNT+1))
    return
  fi
 
  rm -f "$tmpfile"
  echo "Downloading $label ($project/$subj)..."
  if ! curl "${CURLOPTS[@]}" -b "JSESSIONID=$JSESSION" \
      "$XNATHOST/REST/projects/$project/subjects/$subj/experiments/$label/scans/ALL/files?format=zip" \
      -o "$tmpfile"; then
    rm -f "$tmpfile"
  fi
 
  if is_valid_zip "$tmpfile"; then
    if mv "$tmpfile" "$zipfile"; then
      log "OK $project/$subj/$label"
      OK_COUNT=$((OK_COUNT+1))
    else
      echo "  -> FAILED: could not finalize $zipfile" >&2
      rm -f "$tmpfile"
      log "FAIL_MOVE $project/$subj/$label"
      FAIL_COUNT=$((FAIL_COUNT+1))
    fi
    return
  fi
 
  echo "  -> download invalid or unsuccessful; re-authenticating and retrying once..."
  rm -f "$tmpfile"
  authenticate
  if ! curl "${CURLOPTS[@]}" -b "JSESSIONID=$JSESSION" \
      "$XNATHOST/REST/projects/$project/subjects/$subj/experiments/$label/scans/ALL/files?format=zip" \
      -o "$tmpfile"; then
    rm -f "$tmpfile"
  fi
 
  if is_valid_zip "$tmpfile"; then
    if mv "$tmpfile" "$zipfile"; then
      log "OK (retry) $project/$subj/$label"
      OK_COUNT=$((OK_COUNT+1))
    else
      echo "  -> FAILED: could not finalize $zipfile" >&2
      rm -f "$tmpfile"
      log "FAIL_MOVE $project/$subj/$label"
      FAIL_COUNT=$((FAIL_COUNT+1))
    fi
  else
    echo "  -> FAILED: $label"
    rm -f "$tmpfile"
    log "FAIL $project/$subj/$label"
    FAIL_COUNT=$((FAIL_COUNT+1))
  fi
}
 
authenticate
log "START user=$USERID host=$XNATHOST dir=$DOWNLOADDIR project=${PROJECT:-} sessionlist=${SESSIONLIST:-}"
 
if [ -n "${SESSIONLIST:-}" ]; then
  row=0
  n=0
  while IFS=',' read -r proj subj sess extra; do
    row=$((row+1))
    proj=$(clean_field "$proj")
    subj=$(clean_field "$subj")
    sess=$(clean_field "$sess")
    [ -z "$proj$subj$sess${extra:-}" ] && continue
    if [ -n "${extra:-}" ]; then
      echo "Invalid CSV row $row: too many columns" >&2
      log "INVALID_ROW line=$row reason=too_many_columns"
      FAIL_COUNT=$((FAIL_COUNT+1))
      continue
    fi
    if [ -z "$proj" ] || [ -z "$subj" ] || [ -z "$sess" ]; then
      echo "Invalid CSV row $row: expected Project,Subject,Session" >&2
      log "INVALID_ROW line=$row"
      FAIL_COUNT=$((FAIL_COUNT+1))
      continue
    fi
    n=$((n+1))
    echo "[$n]"
    download_session "$proj" "$subj" "$sess"
  done < "$SESSIONLIST"
 
else
  curl "${CURLOPTS[@]}" -b "JSESSIONID=$JSESSION" \
    --get \
    --data-urlencode "xsiType=$XSITYPES" \
    --data-urlencode "format=csv" \
    --data-urlencode "columns=xnat:experimentData/ID,xnat:subjectAssessorData/subject_id,xnat:experimentData/label" \
    "$XNATHOST/REST/projects/$PROJECT/experiments" \
    -o "$CFGFILE" || { echo "Request to CNDA failed." >&2; log "REQUEST_FAIL project=$PROJECT"; exit 1; }
 
  lineCount=$(wc -l < "$CFGFILE")
  if [ "$lineCount" -lt 2 ]; then
    echo "No results returned from CNDA." >&2
    log "NO_RESULTS project=$PROJECT"
    exit 1
  fi
  if grep -qi "<html>" "$CFGFILE"; then
    echo "Your id is probably not enabled for this project, or the project id is wrong." >&2
    log "ACCESS_DENIED project=$PROJECT"
    exit 1
  fi
 
  {
    IFS= read -r header
    # CNDA's REST/experiments CSV endpoint appends its own default ID,label,URI
    # columns after whatever columns are requested, so a real row here is:
    #   xnat:*sessionData/ID, subject_ID, ID, label, URI
    # Field 1 (requested experiment ID) and field 3 (default ID) are the same
    # value; field 4 (default label) is the one we actually want as the label.
    while IFS=',' read -r exptID subjID defaultID label uri extra; do
      exptID=$(clean_field "$exptID")
      subjID=$(clean_field "$subjID")
      label=$(clean_field "$label")
      [ -z "$exptID$subjID$label" ] && continue
      echo "[$((OK_COUNT+FAIL_COUNT+SKIP_COUNT+1))]"
      download_session "$PROJECT" "$subjID" "$label"
    done
  } < "$CFGFILE"
fi
 
log "DONE ok=$OK_COUNT fail=$FAIL_COUNT skip=$SKIP_COUNT"
echo "Done. OK=$OK_COUNT  FAIL=$FAIL_COUNT  SKIP=$SKIP_COUNT"
echo "See $LOGFILE for details."
 
[ "$FAIL_COUNT" -gt 0 ] && exit 1
exit 0