#!/bin/bash

# Extensible, yad-based progress bar server.
# Copyright (C)2019 step
# Dual license MIT and GNU GPLv3
Version=1.0.0

# Read markdown documentation at the end of this file.

FIFOIN="$1"
WIN_WIDTH="$2"
DOMAIN="${3:-checksum}"
DOMAIN_FUNCTION_PATH="$4"
WINDOW_ICON="$5"

# file descriptors: 8, 9(init_progress)

BATCH_TIMEOUT=0.5    # seconds
declare -i VERBOSE=0 # 0-2

# window geometry helper
source yad-lib.sh && yad_lib_init && WITH_YAD_LIB=1
[ "$WIN_WIDTH" ] && CWIDTH=$((WIN_WIDTH * 100 / 1333 + 1)) # for 10pt font

_notify() # [--fold] $1-msg {{{1 DEBUG XXX
# Due to the non-scrollable nature of notify-send's output area using this function to
# print lots of messages quickly becomes unwieldy.  This can be somewhat mitigated by
# passing --fold, which will fold identical messages together.
{
  local unique=$RANDOM$RANDOM
  [ --fold = "$1" ] && unset unique && shift
  local msg=$1
  set -- $(caller)
  local lineno=$1
  notify-send -t 0 "$msg"$'\n'"{${FUNCNAME[1]} $lineno {$unique}}"
}

unset_global_arrays() # {{{1 | => $I[] $aF@[]
{
  # unset the global arrays that init_progress will initilize
  unset I ${!aF@} # all variables that start with "aF"
}

init_progress() # $1-rq-id $2-label $3-bar-text $4-realpath $5-cmd {{{1 $I $aF{,lbl,txt,cmd,siz,bar,pid,mon,pos,ppc} <= |
{
  local rq_id="$1" label="$2" text="$3" realpath="$4" cmd="$5"
  local pid fd mon offset k v idx size
  idx=${#aF[@]}
  I+=( $idx )
  aF+=( "$realpath" )
  aFlbl+=( "$label" )
  center_inside_bar text
  aFtxt+=( "$text" )
  aFcmd+=( "$cmd" )
  size=$(stat -c %s -- "$realpath" 2>/dev/null)
  aFsiz+=( "${size:-0}" )
  aFbar+=( "--bar=$label:NORM" )

  echo "$rq_id@rq_id@" > "$TMPD"/$idx
  $cmd "$realpath" >> "$TMPD"/$idx 2> "$TMPD"/$idx.stderr &
  pid=$!
  aFpid+=( $pid )

  # get the file descriptor that $pid holds on $realpath
  # lsof is the reason why we require a realpath
  local fd="$(lsof -p $pid -F n | grep -m1 -xF -B1 "n$realpath")"
  fd=${fd%%?n*}; fd=${fd#f}

  # monitor the read offset of that file descriptor
  mon="/proc/$pid/fdinfo/$fd"
  if [ "$fd" ] && exec 9< "$mon" 2>/dev/null; then
  # DEBUG NOTE: if set -x is enabled comment out ^^^^ 2>/dev/null temporarily
    aFmon+=( "$mon" )
    while read -u9 k v; do [ "$k" = pos: ] && offset=$v && break; done
    aFpos+=( $offset )
    aFppc+=( $((offset * 100 / size)) ) # progress %
    exec 9<&-

  else # $cmd has either failed or finished already
    aFmon+=( "" )
    aFpos+=( $size )
    aFppc+=( 100 )
  fi
}

center_inside_bar() # $1-varname {{{1 $CWIDTH <= |
# center *$varname text in $CWIDTH-character wide bar; pad text with m-spaces
# actual bar width also depends on bar label; see doc/yad-multi-progress-width.md
{
  [ "$CWIDTH" ] || return
  local -n varname=$1
  local pad len
  len=${#varname}
  if (( len < CWIDTH )); then
    len=$(( (CWIDTH - len) / 2 ))
    printf -v pad "% ${len}s" ""
    varname=$pad$varname$pad$p$p
    varname=${varname// /$'\u2003'}
  fi
  varname=${varname:0:CWIDTH}
}

next_offset() # $1-index {{{1
{
  local idx=$1 k v offset=${aFsiz[$1]}"0" # "0" so read error leads to "! Running[idx]"
  while read k v; do [ "$k" = pos: ] && offset=$v && break; done < "${aFmon[$idx]}"
  aFpos[$idx]=$offset
}

show_progress() # {{{1 $I[] $aF@[] <= |
{
  local idx ppc Running=( any ) progress barnum

  # initialize all bars
  for idx in "${I[@]}"; do
      barnum=$(($idx +1))
      printf %s\\n "$barnum:${aFppc[$idx]}" "$barnum:#${aFtxt[$idx]}"
  done

  # while there are running processes update their progress bars
  while [ "$Running" ]; do
    Running=( )
    for idx in "${I[@]}"; do
      barnum=$(($idx +1))
      next_offset "$idx" #2>/dev/null

      ppc=$((${aFpos[$idx]}00 / ${aFsiz[$idx]})) # truncated is ok
      if (( ppc > ${aFppc[$idx]} )); then
        aFppc[$idx]=$ppc

        # update bar; adds +1 to compensate for truncation
        printf %s\\n "$barnum:$((ppc + 1))" "$barnum:#${aFtxt[$idx]}"
      fi

      # check if pid is still running
      [ -e "${aFmon[$idx]}" ] && Running+=( $idx )
    done
  done

  close_dialog
}

close_dialog() # {{{1 $UNIQUE_WINDOW_TITLE <= |
{
  # sometimes twice is needed
  sleep 0.1; pkill -USR1 -f "$UNIQUE_WINDOW_TITLE"
  sleep 0.1; pkill -USR1 -f "$UNIQUE_WINDOW_TITLE"
}

on_cancel_emit_response() # {{{1
{
  trap - HUP INT QUIT TERM ABRT EXIT

  # stop unfinished commands
  kill ${aFpid[@]} 2> /dev/null
  wait ${aFpid[@]} 2> /dev/null

  emit_response
}

emit_response() # {{{1 $I[] <= |
# Emit {stdout,stderr} lines in index order <index>"\t"{"O","E"}<line>,
# and remove intermediate output files.
{
#  _notify "I(${I[*]})" #XXX
  ! (( ${#I[*]} )) && return
  [ "${TMPD#${TMPDIR:-/tmp}/}" ] && [ -d "$TMPD" ] || return 1
#  _notify "`ls $TMPD`" #XXX

  local s i p prefix line
  set --
  for s in O: E:.stderr; do
    for i in "${I[@]}"; do
      p="$TMPD/$i${s#*:}" prefix="$i"$'\t'"${s%%:*}"
#      if ! [ -s "$p" ]; then _notify "zero size $p"; fi #XXX
      if [ -s "$p" ]; then
        while read line; do
          echo "$prefix$line"
        done < "$p"
      fi
      set -- "$@" "$p"
    done
  done

  rm -f "$@"
}

on_exit() # {{{1
{
  trap - HUP INT QUIT TERM ABRT

  on_cancel_emit_response

  rm -fr "$TMPD"* "$FIFOIN"

  exit 128
}

do_batch() # [$1-max-parallel-computations] {{{1 $I{} $BATCH_TIMEOUT $DOMAIN <= | => $UNIQUE_WINDOW_TITLE $I[]
# Batch input ends after $BATCH_TIMEOUT seconds without input or on reading an empty line.
# do_batch expects a domain function to read domain-specific data from the input fifo.
{
  local max_par_comp="$1"
  # maximum parallel computations upper limits the number of progress bars per dialog.
  # values: 0(unlimited, not recommended), ""(auto), integer > 0
  # auto is computed as four times the number of cores, not to exceed 16
  if ! [ "$max_par_comp" ]; then
    max_par_comp=$(( 4 * $(getconf _NPROCESSORS_ONLN) ))
    (( max_par_comp > 16 )) && max_par_comp=16
  fi
  (( VERBOSE > 0 )) && echo "MAX_PAR_COMP($max_par_comp)" >&2

  printf -v UNIQUE_WINDOW_TITLE '%s\t\t\t%s' "$DOMAIN" "$RANDOM$RANDOM"
  local char first_line in_batch=1 par_comp_count Running idx xid

  # block waiting for batch input to start
  read -r -u8 -N1 char || return 1

  # reset cancel flag
  rm -f "$TMPD"/cancel

#  _notify "enter in_batch" #XXX
  # Unblocking read until max_par_comp exceeded or timeout or empty line
  while (( in_batch )); do

    unset_global_arrays
    par_comp_count=0
#    _notify --fold "enter in_par_comp" #XXX

    while (( 0 == max_par_comp || par_comp_count++ < max_par_comp )); do

      # On timeout
      # (bash ignores timeout if reading from a regular file)
      # (null IFS otherwise any space starting in column 2 would be chopped)
      ! IFS=$'\0' read -r -u8 -t $BATCH_TIMEOUT first_line && in_batch=0 && break

      # On empty line
      ! [ "$char$first_line" ] && in_batch=0 && break

      # Compute a record; it sinks output if user pressed Cancel at @43
      ${DOMAIN}_record "$char$first_line"
      unset char

    done # in_par_comp
#    _notify --fold "done in_par_comp" #XXX

    # At this point up to $max_par_comp computations are running.
    # Start a progress dialog unless all of them have already ended.

    # check which pids are still running
    Running=( )
    for idx in "${I[@]}"; do
      [ -e "${aFmon[$idx]}" ] && Running+=( $idx )
    done
    (( VERBOSE > 0)) && printf >&2 "% 2d %s: %s\n" \
      "${#I[*]}" started "${I[*]}" "${#Running[*]}" running "${Running[*]}"

    if (( ${#Running[*]} )); then

      # Try to center the sub-dialog over the client window.
      if [ "$WITH_YAD_LIB" ] && [ -r "$FATDOG_PROGRESS_XID_PATH" ]; then
        read xid < "$FATDOG_PROGRESS_XID_PATH"
        [ "$xid" ] && yad_lib_set_YAD_GEOMETRY "$xid" "" "90:50:$WIN_WIDTH:-1:$WIN_WIDTH:-1"
      fi

      show_progress |

      {
        trap on_cancel_emit_response HUP INT QUIT TERM ABRT

        yad \
          --title="$UNIQUE_WINDOW_TITLE" \
          ${WINDOW_ICON:+--window-icon=$WINDOW_ICON} \
          ${YAD_GEOMETRY_POPUP:- --center} \
          --no-focus \
          --undecorated \
          --borders=10 \
          --buttons-layout=center \
          --button=gtk-cancel:"sh -c \"echo @43 >'$TMPD'/cancel; kill -USR1 \$YAD_PID\"" \
          --multi-progress \
            --align=left \
            "${aFbar[@]}" \
        &

        2>/dev/null wait # very important
      }
    fi

    # Send client the responses from computations started while in_par_comp
    emit_response

  done # in_batch
#  _notify "done in_batch" #XXX
}

checksum_record() # $1-record-first-line {{{1 $BATCH_TIMEOUT <= |
# Batch input ends after $BATCH_TIMEOUT seconds without input or on reading an empty line.
# A domain record function expects to read domain-specific data from the input fifo.
# A domain record function starts the computation involving the record.
{
  # the 'checksum' domain record consists of
  local rq_id="$1" label realpath digest cmd

  # rq_id is given, now read the rest of the domain-specific data
  read -r -u8 -t $BATCH_TIMEOUT label    &&
  read -r -u8 -t $BATCH_TIMEOUT realpath &&
  read -r -u8 -t $BATCH_TIMEOUT digest   &&
  read -r -u8 -t $BATCH_TIMEOUT cmd      &&
  : "domain reads($?)"

  # start the computation unless cancelled
#  if [ -e "$TMPD"/cancel ]; then _notify "cancelled!"; fi #XXX
  if ! [ -e "$TMPD"/cancel ]; then
#    _notify "starting ($cmd) ($realpath)" #XXX
    init_progress "$rq_id" "$label" "$digest" "$realpath" "$cmd"
  fi
}

# Main {{{1

TMPD=$(mktemp -d -p "${TMPDIR:-/tmp}" "${0##*/}_XXXXXX")
chmod 700 "$TMPD" || exit 1
trap on_exit EXIT HUP INT QUIT TERM ABRT

[ -e "$DOMAIN_FUNCTION_PATH" ] && source "$DOMAIN_FUNCTION_PATH"

if [ -p "$FIFOIN" ]; then
  printf "$(gettext "%s: Warning: '%s' already exists. This could be a sign that the client writes before the server is ready to read.")\n" "${0##*/}" "$FIFOIN" >&2
fi
rm -f "$FIFOIN" # could be a regular file, if existing
mkfifo "$FIFOIN" && exec 8< "$FIFOIN" || exit 1

while true; do
  if (( VERBOSE > 1 )); then
    printf "waiting for new batch at " >&2
    date >&2
  fi

  unset_global_arrays
  if ! do_batch "$FATDOG_PROGRESS_MAX_PAR_COMP"; then
    exec 8< "$FIFOIN" || exit 1
  fi
done

exit

: << 'MARKDOWNDOC' #{{{1
## Usage

    $0 /path/to/pipe [bar_width] [domain [/path/to/domain-functions.sh] [window_icon]]

Default domain: `checksum` (functions are included in this script).

## Environment

* `FATDOG_PROGRESS_MAX_PAR_COMP` : maximum parallel computations, which upper
  limits the number of progress bars that will be shown in the dialog.
  * Values: 0(unlimited, not recommended), ""(auto), integer > 0.
  * Auto is computed as four times the number of cores, not to exceed 16.

* `FATDOG_PROGRESS_XID_PATH` : Path of a file that contains the X window id (XID)
  of the client's main window. A yad client can create such file with
  `yad --print-xid=FILEPATH`. If set, The server will try to center its progress
  dialog window over the X Window identified by the XID.

## Description

The server will create the named pipe and remove it on exit.
Kill `-SIGINT` the server to terminate it cleanly.

While active, the server reads requests from the named pipe and outputs
responses on stdout.  While a request is being processed, the server displays a
progress bar for that request. Requests are automatically batched together, so
that all progress bars of the same batch will display together in the same
dialog. A batch terminates when either `$FATDOG_PROGRESS_MAX_PAR_COMP` is
exceeded or `$BATCH_TIMEOUT` expires without reading a request or an empty line
is read.

A request record consists of multiple lines, which represent some domain-
specific computation and its arguments. and starts with a client request id on
a line by itself. The client can set any value for the request id, which is
returned with the response record, and can be used to match a request/response
pair.  For instance, for the default `checksum` domain, a request consists of
five lines - request id, label, real path, digest and command. A batch of two
requests could be:

```
1
this_file
/real/path/to/this_file
MD5
md5sum
2
this_file
/real/path/to/this_file
SHA256
sha256sum

```

ending with an empty line or with a timeout as explained above.

Note that only **real** paths are supported. So you need to resolve links and
mounts with `realpath(1)` or `readlink(1)` before sending them to the server.
This is a requirement for all domains.  Note also that the computations of all
batched requests take place concurrently in parallel.

A response record consists of multiple lines.  Each line is prefixed with a
zero-based index, which tracks the request input order within the current
batch, then the tab character `^I` then either `O` or `E`, respectively for
standard output or standard error of the domain command, then:
* for the first response line, the request id followed by `@rq_id@`;
* for other response lines, a standard output/error line from the domain.

All `E` lines, if any, come after all `O` lines, if any.
If a domain computation terminates prematurely, due to errors or signals, its
response may consist of the `@rq_id@` line only.

Note that responses are sorted by index, therefore by input order, not by
computation time.

### Look

By default the progress dialog is centered on screen, and progress bar width is
unspecified. If you specify `bar_width` on the command-line, the server will
try to make all bars `bar_width` pixels wide - approximately - see
doc/yad-multi-progress-width.md for details. If environment variable
`FATDOG_PROGRESS_XID_PATH` is set, the server will try to center progress
bars over the window that that variable references.

### Extending the server

The server sources the `domain-functions.sh` file, which must define function
`<domain>_record`, e.g. `checksum_record`, and can also define other functions.
The domain record function is tasked with reading a request record from the named
pipe, and initializing its computation via function `init_progress`.
Essentially, the domain record function translates request arguments
into the elements that constitute a progress bar, namely: a display label, an
inside text, and a background process that reads a file real path.

The domain record function reads input lines from file descriptor `8`. Before
writing your own domain record function please study function `checksum_record`
for guidance.

### How does it work

In a nutshell.  The Linux kernel knows the current read position of all file
descriptors open for reading.  While a computation takes place, the server
monitors the file read position and divides it by the file size to get a
percentage, which is then fed to a yad progress-bar control.

## FAQ

Q. The server doesn't show a progress bar for my input set. Why?

A. Most likely by the time the server is ready to display the progress barѕ all
computations in the input set have already ended, therefore the server skips
displaying the progress dialog.

MARKDOWNDOC

