#!/bin/bash

function usage {
	echo "$0 usage:"
	echo " Download atcf fixes from https://www.nrlmry.navy.mil/atcf_web/docs/.fixes/<year>"
	echo " <-h|--help : this help"
	echo "  $0 [-d dir] [--nrt] [--force] [--no-recover] [start_year [stop_year]]"
        echo "    -v : verbose. -vv more verbose"
        echo "    -d <dir> : download file to <dir>. default to script dir"
        echo "    --nrt : dowload active storm only"
        echo "    --force : force wget to redownload files (i.e without '-N' option)"
        echo "    --no-recover : do not try to recover from previous logged errors"
        echo "    start_year: YYYY. default to now"
        echo "    stop_year: YYYY. default to now"
}

GETOPT=$(getopt -n $0 -o hdv --long help,nrt,force -- "$@")

[ $? -eq 0 ] || usage
eval set -- "$GETOPT"

nrt="false"
force_year="false"
force="false"
recover="true"
start_year=$(date -d 'now - 15 days' +%Y)  # to handle year change
stop_year=$(date -d 'now' +%Y)
verbose="false"
dest_dir=$(dirname $0)
wget_opts=""

url=https://www.nrlmry.navy.mil/atcf_web

function logger {
    [ "$verbose" == "true" ] && echo $1 >&2
}

function get_year_files {
	# Retrieve file list from, for example https://www.nrlmry.navy.mil/atcf_web/docs/.fixes/2022
	# Only complete files are included, not ones containing only one day of data
	files_url=$url/docs/.fixes/$1
	logger "get files from $files_url"
	wget --no-check-certificate -q -O- $files_url | grep -oP '<a href=".+?">\K.+?(?=<)' | grep  'f..[01234567].....\.dat' | xargs
	return "${PIPESTATUS[0]}"
}

function get_nrt_files {
    storms_url=$url/stormlist.current
    logger "get nrt storms from $storms_url"
    wget --no-check-certificate -q -O- $storms_url | grep "^..[01234567]" | xargs -ISTORM echo fSTORM.dat | xargs
    return "${PIPESTATUS[0]}"
}

function check_file {
    status=0
    file=$1

    backup=$1.1

    for f in $file $backup ; do
        if [  -f $f ] && grep -q "<html>" $f ; then
            rm -f $f
            status=1
        fi
    done

    if [ -f $file ] && [ -f $backup ] ; then
        # keep the largest file
        s_file=$(stat -c%s $file)
        s_backup=$(stat -c%s $backup)
        if ((s_backup > s_file)) ; then
            logger "$backup size > $file size: restoring backup"
            mv -f $backup $file
            status=1
        else
            rm $backup
        fi
    elif [ ! -f $file ] ; then
        status=1
    fi

    return $status
}


function download {
    # download b????YYYY.dat'
    # downoad remote file to year directory, with some checks

    file_dir=$(dirname $1)
    file=$(basename $1)
    logger "to download : $1"

    file_year=$(echo $file | cut -c 6-9)
    previous_ts=0
    new_ts=0
    if [ -f "$file_dir/$file" ] ; then
        previous_ts=$(date -r "$file_dir/$file" +%s)
    fi


    mkdir -p $file_dir

    (
        cd $file_dir

        check_file $file

        file_url=$url/docs/.fixes/$file_year/$file
        wget --no-check-certificate $wget_opts --backups=1 $file_url

        check_file $file
    )
    status=$?

    # manualy check timestamp to get correct status message
    if [ -f "$file_dir/$file" ] ; then
        new_ts=$(date -r "$file_dir/$file" +%s)
    fi
    if [ $status -eq 0 ] ; then
        if (( previous_ts == 0)) && (( new_ts > 0 )) ; then
            echo "created"
        elif (( previous_ts  == new_ts )) ; then
            echo "up to date"
        elif (( new_ts > previous_ts)) ; then
            echo "updated"
        elif (( new_ts < previous_ts)) ; then
            echo "updated (corrected timestamp)"
        fi
    else
        if (( new_ts != 0 )) && (( previous_ts != 0 )) ; then
            echo "download error. old version kept"
        elif (( new_ts == 0 )) && (( previous_ts == 0 )) ; then
            echo "download error. no previous backup"
        else
             echo "error"
        fi

    fi


    return $status
}

verbose_level=0

while true; do
    case "$1" in
        --nrt ) nrt="true" ; shift ;;
        --force ) force="true" ; shift ;;
        --no-recover ) recover="false" ; shift ;;
        -v )  ((verbose_level++)) ; shift;;
        -h | --help ) usage ; exit 0 ;;
        -d ) dest_dir=$1 ; shift ;;
        -- ) shift ; break ;;
    esac
done

cd $dest_dir

if [ "$verbose_level" != "0" ] ; then
    verbose="true"
fi

if ((verbose_level > 1))  ; then
    wget_opts="$wget_opts -nv"
else
    wget_opts="$wget_opts -q"
fi

if [ "$force" == "false" ] ; then
    wget_opts="$wget_opts -N"
fi

if [ -n "$1" ] ; then
    start_year=$1
    force_year="true"
fi

if [ -n "$2" ] ; then
    stop_year=$2
fi


recover_files=""
if [ "$recover" == "true" ] &&  [ -s errors.txt ] ; then
    recover_files=$(cat errors.txt | xargs)
    echo "Will try to recover those files: $recover_files"
fi


nrt_files=""
if [ "$nrt" == "true" ] ; then
    nrt_files=$(get_nrt_files)
    if [ $? -ne 0 ] ; then
        echo "error fetching nrt files" >&2
        exit 2
    fi
else
    force_year="true"
fi

all_files="$nrt_files $recover_files"

years=""
if [ "$force_year" == "true" ] ; then
    years=$(seq $start_year $stop_year)
fi

for year in $years ; do
    all_files="$all_files $(get_year_files $year)"
    if [ $? -ne 0 ] ; then
        echo "error fetching index files for year $year" >&2
        exit 2
    fi

done


all_files=($all_files)

count=${#all_files[@]}

success=""
errors=""
count_success=0
count_errors=0

for i in "${!all_files[@]}" ; do
    file="${all_files[$i]}"
    echo  -n "$((i + 1))/$count $file ... "
    file_year=$(echo $file | cut -c 6-9)

    msg=$(download "$file_year/$file")
    status=$?
    echo $msg
    if [ $status -ne 0 ] ; then
        errors="$errors $file"
        echo $file >> errors.txt
        cat errors.txt | sort | uniq | sponge errors.txt
        ((count_errors++))
    else
        success="$success $file"
        [ -f errors.txt ]  && grep -v $file errors.txt | sponge errors.txt
        ((count_success++))
    fi
done

if [ $count_errors -gt 0 ] ; then
    echo "errors (logged to $(readlink -f $PWD)/errors.txt : $errors"
    echo "$count_errors/$count errors"
else
    echo "no fatal errors reported"
fi

exit $count_errors



