#!/bin/bash
#
# Kexec setup
# creates new initrd for the crash kernel and updates the 
# kernel command line parameters to load the crash kernel
# Copyright 2005 Red Hat, Inc.
# Copyright (c) 2015-2021 by Cisco Systems, Inc.
# All rights reserved.

source /etc/init.d/spirit_pd.sh
source /etc/init.d/functions

KEXEC=/usr/sbin/kexec
KDUMP_COMMANDLINE=`cat /proc/cmdline`
KDUMP_INITRD="/boot/crash-initrd.img"
KDUMP_KERNEL=""
LOGGER="/usr/bin/logger -p info -t kdump"
ARCH=`arch`
if [ "$ARCH" == "aarch64" ]
then
  KDUMP_COMMANDLINE_APPEND="maxcpus=1 reset_devices cgroup_disable=memory"
else
  KDUMP_COMMANDLINE_APPEND="irqpoll maxcpus=1 reset_devices cgroup_disable=memory"
fi
dumpdisk=""

function make_crash_initrd
{
  crashfs=$(mktemp -d /tmp/crashfs.XXXXXX)
  cd $crashfs
  if [ "$ARCH" == "x86_64" ] || [ "$ARCH" == "aarch64" ]
  then  
    lib_type=lib64
  else
    lib_type=lib
  fi
  if [ "$ARCH" == "aarch64" ]
  then
    mkdir -p $lib_type lib $lib_type/modules bin sbin tmp usr/bin usr/lib\
    usr/sbin var/log 
  else
    mkdir -p $lib_type $lib_type/modules bin sbin tmp usr/bin usr/lib\
    usr/sbin var/log 
  fi
  BINARIES=('busybox' 'bash' 'makedumpfile' 'pcimemread' 'pcimemwrite'
  'stat' 'lspci' 'arch')

  # PD may have more binaries to package, this adds those into
  # the infra for picking libs
  if [ "$(type -t platform_binaries_set)" = "function" ]; then
      BINARIES=(${BINARIES[@]} $(platform_binaries_set))
  fi

  if [ -n "$dumpdisk" ]
  then
    BINARIES=(${BINARIES[@]} 'lsblk' 'lvm' 'insmod')
  fi

  for prog in "${BINARIES[@]}"
  do
    #Find the library dependencies 
    libs=$(ldd `which $prog` | awk 'BEGIN{ORS=" "}$1\
    ~/^\//{print $1}$3~/^\//{print $3}'\
    | sed 's/,$/\n/')

    #Copy the library dependencies
    for lib in $libs
    do
      if [ "$ARCH" == "aarch64" ]
      then    
          if [[ $lib == *"lib64"* ]]
          then
              cp $lib $lib_type/
          else
              cp $lib lib/
          fi
      else
          cp $lib $lib_type/
      fi
    done    
  done

# copy binaries specific to crash kernel collection of vmcore

  if [ -n "$dumpdisk" ]
  then
    # Copying all required modules
    if [ "$(type -t platform_copy_required_kernel_crash_modules)" = "function" ]; then
        platform_copy_required_kernel_crash_modules $lib_type/modules
    fi

    cp /usr/bin/lsblk usr/bin
    cp /usr/sbin/lvm usr/bin/
    cp /sbin/insmod sbin/insmod
  else
    cp /sbin/mount.nfs sbin/
  fi
  
  #Copying required binaries
  cp /bin/busybox bin/
  cp /bin/bash bin/
  cp /usr/bin/makedumpfile usr/bin/
  cp /usr/sbin/pcimemread usr/sbin/
  cp /usr/sbin/pcimemwrite usr/sbin/
  cp /usr/sbin/lspci usr/sbin/
  cp /bin/stat bin/
  cp /usr/bin/arch usr/bin/
  cp /usr/bin/tr usr/bin/
  cp /bin/date bin/   # BusyBox version doesn't support %N option
  cp /bin/sleep bin/  # BusyBox version doesn't support decimal number

  #Linking binaries to busybox
  ln -s /bin/bash bin/sh
  ln -s /bin/busybox bin/awk
  ln -s /bin/busybox bin/cat
  ln -s /bin/busybox bin/chown
  ln -s /bin/busybox bin/chmod
  ln -s /bin/busybox bin/df
  ln -s /bin/busybox bin/flock
  ln -s /bin/busybox bin/grep
  ln -s /bin/busybox bin/hostname
  ln -s /bin/busybox bin/ifconfig
  ln -s /bin/busybox bin/ln
  ln -s /bin/busybox bin/ls
  ln -s /bin/busybox bin/mkdir
  ln -s /bin/busybox bin/modprobe
  ln -s /bin/busybox bin/mount
  ln -s /bin/busybox bin/mv
  ln -s /bin/busybox bin/od
  ln -s /bin/busybox bin/ping
  ln -s /bin/busybox bin/ps
  ln -s /bin/busybox bin/readlink
  ln -s /bin/busybox bin/realpath
  ln -s /bin/busybox bin/rm
  ln -s /bin/busybox bin/rmdir
  ln -s /bin/busybox bin/sed
  ln -s /bin/busybox bin/sort
  ln -s /bin/busybox bin/tail
  ln -s /bin/busybox bin/tar
  ln -s /bin/busybox bin/touch
  ln -s /bin/busybox bin/umount
  ln -s /bin/busybox bin/wc
  ln -s /bin/busybox bin/which
  ln -s /bin/busybox bin/zcat

  ln -s /bin/busybox sbin/sysctl
  ln -s /bin/busybox usr/bin/basename
  ln -s /bin/busybox usr/bin/cut
  ln -s /bin/busybox usr/bin/dirname
  ln -s /bin/busybox usr/bin/find
  ln -s /bin/busybox usr/bin/head
  ln -s /bin/busybox usr/bin/logger
  ln -s /bin/busybox usr/bin/mkfifo

  mkdir -p etc/rc.d/init.d/mod_ins
  ln -s /etc/rc.d/init.d etc/init.d
  # Copy Mandatory PD scripts
  cp /etc/rc.d/init.d/calvados_bootstrap.cfg etc/rc.d/init.d/
  cp /etc/rc.d/init.d/spirit_pd.sh etc/rc.d/init.d/
  cp /etc/rc.d/init.d/pd-functions etc/rc.d/init.d/
  cp /etc/fstab etc/
  # Copy FPGA related PD scripts for identifying platform info (method used
  # in platforms like Fretta)
  if [ -f /pkg/etc/get_card_inst.sh ]; then
      cp /pkg/etc/get_card_inst.sh etc/rc.d/init.d/
  fi
  # Copy FPGA related PD scripts for identifying platform info via Platform
  # ID and FPGA ID on platform supporting FPGA implmented via IP blocks
  mkdir -p usr/local/etc
  if [ -f /usr/local/etc/fpga-functions ]; then
      cp /usr/local/etc/fpga-functions usr/local/etc/
  fi
  if [ -f /usr/local/etc/fpga_platform_data_config ]; then
      cp /usr/local/etc/fpga_platform_data_config usr/local/etc/
  fi

  if [ -f /usr/local/etc/functions.platform ]; then
      cp /usr/local/etc/functions.platform usr/local/etc/
  fi

  if [ -f /usr/local/etc/functions.pci ]; then
          cp /usr/local/etc/functions.pci usr/local/etc/
  fi

  # Copy Optional PD scripts
  if [ -f /usr/local/etc/doorbell_event_defs ]; then
      cp /usr/local/etc/doorbell_event_defs usr/local/etc/
  fi
  if [ -f /usr/local/etc/send_doorbell_event.sh ]; then
      cp /usr/local/etc/send_doorbell_event.sh usr/local/etc/
  fi

  # Copy more PD specific bins, libs and scripts into location
  if [ "$(type -t platform_copy_bins_libs)" = "function" ]; then
      platform_copy_bins_libs `pwd`
  fi

  # The following two APIs are used by Fretta for now. Should be removed
  # once Fretta loads kernel modules through platform API
  [ -f /etc/rc.d/init.d/load_kernel_modules.sh ] && \
    cp /etc/rc.d/init.d/load_kernel_modules.sh etc/rc.d/init.d/

  [ -f /etc/rc.d/init.d/mod_ins/module-load-functions ] && \
    cp /etc/rc.d/init.d/mod_ins/module-load-functions etc/rc.d/init.d/mod_ins

  cat << 'EOF' > init
#!/bin/sh
# This is a snippet from initramfs-framework.
# 

PATH=/sbin:/bin:/usr/sbin:/usr/bin
COREDIR=""
CORENAME=""
DUMPDISK=""

# SCRATCHDIR=/misc/scratch is OK, even if the normal XR mount point
# for the kernel dumps is not /misc/scratch, since the name is basically
# arbitrary - the target partition can be mounted using any name.
SCRATCHDIR=/misc/scratch
LOGGER="/usr/bin/logger -s -p info -t kdump"
SLOT=""

function nfs_make_core_file
{
  CORENAME=core.slot${SLOT}
  $LOGGER "Starting kdump file ${COREDIR}/${CORENAME}"
  # Notify active RP about KDUMP started event
  if [ "$(type -t pd_notify_os_kdump_started)" = "function" ]; then
    pd_notify_os_kdump_started "Starting kdump file ${COREDIR}/${CORENAME}"
  fi

  START_KCORE=$(date +%s)
  makedumpfile --message-level 4 -d 17,31 /proc/vmcore ${COREDIR}/${CORENAME}.incomplete
  retval=$?
  TSTAMP=`stat -c "%x" ${COREDIR}/${CORENAME}.incomplete | awk -F '[-| |:|.]' '{print $1$2$3"-"$4$5$6}'`
  END_KCORE=$(date +%s)
  if [ "$retval" == "0" ];then
    mv ${COREDIR}/${CORENAME}.incomplete ${COREDIR}/kernel.${TSTAMP}.${CORENAME}.kdump
    $LOGGER "kdump core finished:took $(( $END_KCORE - $START_KCORE )) secs"
    $LOGGER "Kernel core copied to $COREDIR/kernel.$TSTAMP.$CORENAME.kdump. The core will be available once boot is completed !!!"
  else
    $LOGGER "makedumpfile failed, ${COREDIR}/kernel.${TSTAMP}.${CORENAME}.INCOMPLETE.kdump is INCOMPLETE"
    mv ${COREDIR}/${CORENAME}.incomplete ${COREDIR}/kernel.${TSTAMP}.${CORENAME}.INCOMPLETE.kdump
    if [ "$(type -t pd_notify_os_kdump_create_failed)" = "function" ]; then
        pd_notify_os_kdump_create_failed "makedumpfile faile to create ${COREDIR}/kernel.${TSTAMP}.${CORENAME}.INCOMPLETE.kdump"
    fi
  fi

  if [ "$(type -t pd_notify_os_kdump_ended)" = "function" ]; then
      pd_notify_os_kdump_ended "Kernel core copied to $COREDIR/kernel.$TSTAMP.$CORENAME.kdump"
  fi
}

function copy_core_nfs_wrapper
{
  COREDIR="/RP"
  mkdir -p ${COREDIR}
  if [ "$(type -t platform_nfs_mount_core_dir)" = "function" ]; then
      platform_nfs_mount_core_dir
  fi

  if [ "$?" == "0" ]; then
    nfs_make_core_file
    umount -l ${COREDIR}
  fi
}

function copy_core_dumpdisk
{
  if [ "$(type -t platform_load_required_kernel_crash_modules)" = "function" ]; then
      platform_load_required_kernel_crash_modules
  fi

  # PD specific configuration for punching wdogs, peer nodes, etc
  if [ "$(type -t platform_configuration)" = "function" ]; then
      platform_configuration
  fi

  echo -n "waiting for disk to come up"
  # lsblk -dno SUBSYSTEMS
  # 
  # block:scsi:usb:pci    <- usb thumb drive
  # block:scsi:pci        <- sata drive (fixed, distributed)
  # block:nvme:pci        <- nvme drive (centralized)
  while [ "`lsblk -dno SUBSYSTEMS | grep -v usb`" == "" ]; do
    echo -n .
    sleep 1
  done
  echo

  lvm vgscan --mknodes --ignorelockingfailure >/dev/null 
  lvm vgchange -a y --monitor n --ignorelockingfailure >/dev/null 
  mkdir -p $SCRATCHDIR
  mount $DUMPDISK $SCRATCHDIR >/dev/null 2>/dev/null 
  if [ $? == "0" ] ; then
    COREDIR=${SCRATCHDIR}/core
    mkdir -p ${COREDIR} > /dev/null 2>/dev/null
    TSTAMP="`date +"%Y%m%d-%H%M%S"`"
    CORENAME=kernel.${TSTAMP}.core.slot${SLOT}
    START_KCORE=$(date +%s)
    echo $(date) ":Amount of free space on target file system for kdump" > $COREDIR/$CORENAME.kdump.txt
    df -h ${COREDIR} >> $COREDIR/$CORENAME.kdump.txt
    $LOGGER "Starting kdump file ${CORENAME}"
    # Notify active RP about KDUMP started event
    if [ "$(type -t pd_notify_os_kdump_started)" = "function" ]; then
        pd_notify_os_kdump_started "Starting kdump file $COREDIR/$CORENAME"
    fi
    if [[ $(arch) == "aarch64" ]]; then
      makedumpfile -c --message-level 4 -d 31 /proc/vmcore /$COREDIR/$CORENAME.incomplete &
      dump_pid=$!
    else
      makedumpfile -c --message-level 4 -d 19,31 /proc/vmcore /$COREDIR/$CORENAME.incomplete &
      dump_pid=$!
    fi
    while ps | grep "$dump_pid" | grep -v "grep" > /dev/null 2>&1
    do
        sleep 5
        echo -n $(date) ":Size of core file is now " >> $COREDIR/$CORENAME.kdump.txt
        ls -lh /$COREDIR/$CORENAME.incomplete | awk '{print $5}' >> $COREDIR/$CORENAME.kdump.txt 2>/dev/null
    done
    wait $dump_pid
    retval=$?
    END_KCORE=$(date +%s)
    if [ "$retval" == "0" ]; then
      #Using mv to make sure the core file is complete
      mv /$COREDIR/$CORENAME.incomplete /$COREDIR/$CORENAME.kdump
      $LOGGER "kdump core finished:took $(( $END_KCORE - $START_KCORE )) secs"
      $LOGGER "Kernel core copied to $COREDIR/$CORENAME.kdump. The core will be available once boot is completed !!!"
      if [ "$(type -t pd_notify_os_kdump_ended)" = "function" ]; then
          pd_notify_os_kdump_ended "Kernel core copied to /$COREDIR/$CORENAME.kdump"
      fi
    else
      $LOGGER "makedumpfile failed, /$COREDIR/$CORENAME.INCOMPLETE.kdump is INCOMPLETE"
      mv /$COREDIR/$CORENAME.incomplete /$COREDIR/$CORENAME.INCOMPLETE.kdump
      if [ "$(type -t pd_notify_os_kdump_create_failed)" = "function" ]; then
          pd_notify_os_kdump_create_failed "makedumpfile failed to create /$COREDIR/$CORENAME.INCOMPLETE.kdump"
      fi
    fi
    umount $SCRATCHDIR
  else
    $LOGGER "Failed to mount $DUMPDISK"
    if [ "$(type -t pd_notify_os_kdump_mount_failed)" = "function" ]; then
        pd_notify_os_kdump_mount_failed "$DUMPDISK"
    fi
  fi
}

cd /
# initialize /proc, /sys and /var/lock
mkdir -p /proc /sys /var/lock /dev

mount -t sysfs sysfs /sys
mount -t proc /proc /proc
mount -t devtmpfs devtmpfs /dev

echo "Running INIT process in Crash Kernel"

DUMPDISK=`cat /proc/cmdline | awk -F "dumpdisk=" '{ print $2}' | cut -d " " -f1`

. /etc/rc.d/init.d/spirit_pd.sh

 if [ "$(type -t pd_start_bin)" = "function" ]; then
      pd_start_bin
 fi 

# Get the slot nodeid string
if [ "$(type -t pd_get_my_int_nodeid_str)" = "function" ]; then
    SLOT=`pd_get_my_int_nodeid_str`
    # Check if there is failure condition, as with failure we cannot use the
    # value from the funtion output due to error messages.
    if [ $? -ne 0 ]; then
        # Default to RP0 under error condition
        SLOT=0_RP0
    fi
else
    SLOT=0_RP0
fi

# Reconfigure the watchdog to timeout in 300 seconds, this should be enough
# to create the core dump, but for arm keeping 420 seconds, as it takes 
# more time.
if [ "$(type -t pd_cfg_watchdog_timeout)" = "function" ]; then
    if [[ $(arch) == "aarch64" ]]; then
        pd_cfg_watchdog_timeout 420 
    else
        pd_cfg_watchdog_timeout 300
    fi
fi

if [ -n "$DUMPDISK" ]; then
  copy_core_dumpdisk
else
  copy_core_nfs_wrapper
fi

# delay for the post code to be propagated
sleep 1
# reboot system
$LOGGER "Rebooting system"
if [ "$(type -t pd_board_reset)" = "function" ]; then
    pd_board_reset
fi

# delay for the reset to be triggered, init could exit causing another panic
sleep 10
# Emergency Reboot in case above reset fails
echo b > /proc/sysrq-trigger

EOF
  
  chmod 700 init
  find . | cpio --quiet -o --format='newc' | gzip -9 > /boot/crash-initrd.img
  rm -rf $crashfs
}


function load_kdump()
{
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/crashkernel=[^ ]*//'`
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/mem=[0-9]\+[GMKgmk]* *//'`
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/bigphysarea=[^ ]*//'`
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's;rootfs-thin-lv;root=/dev/ram rootfs-thin-lv;'`
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/printk.always_kmsg_dump=1 //'`
  KDUMP_COMMANDLINE="$KDUMP_COMMANDLINE slot=${SLOT}"
  KDUMP_COMMANDLINE="${KDUMP_COMMANDLINE} ${KDUMP_COMMANDLINE_APPEND}"
  [ "$ARCH" == "x86_64" ] && load_kdump_x86_64
  [[ "$ARCH" =~ "arm" ]] && load_kdump_arm
  [[ "$ARCH" == "aarch64" ]] && load_kdump_aarch64

$KEXEC -p $KEXEC_ARGS $KDUMP_KERNEL --initrd=$KDUMP_INITRD \
--command-line="$KDUMP_COMMANDLINE" 2>/dev/null
if [ $? == 0 ]; then
 $LOGGER "Loaded kdump kernel"
 return 0
else
 $LOGGER "Failed to load kdump kernel: ret=$?"
 return 1
fi
}

function load_kdump_x86_64()
{
  KDUMP_KERNEL=/boot/bzImage
  if [ -n "${dumpdisk}" ]; then
    KDUMP_COMMANDLINE="${KDUMP_COMMANDLINE} dumpdisk=${dumpdisk}"
  fi
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's;aer=off;pci=noaer;'`
}

function load_kdump_arm()
{
  KEXEC_ARGS="$KEXEC_ARGS --atags --type zImage"

  KDUMP_KERNEL=/boot/zImage
    
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/vmalloc=[^ ]*//'`
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/pci_errors_enable //'`
}

function load_kdump_aarch64()
{
  KDUMP_KERNEL=/boot/Image
  if [ -n "${dumpdisk}" ]; then
    KDUMP_COMMANDLINE="${KDUMP_COMMANDLINE} dumpdisk=${dumpdisk}"
  fi
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/rootflags=i_version //'`
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/slab_nomerge vsyscall=none //'`
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's/raid=noautodetect //'`
  KDUMP_COMMANDLINE=`echo $KDUMP_COMMANDLINE | sed -e 's;aer=off;pci=noaer;'`
}
function check_kernel_parameter()
{
  [ `cat /sys/kernel/kexec_crash_size` -eq 0 ] && return 1
  return 0
}


function status()
{
  [ ! -e /sys/kernel/kexec_crash_loaded ] && return 2
  [ `cat /sys/kernel/kexec_crash_loaded` -eq 1 ]  && return 0
  return 1
}

function set_kernel_panic_triggers()
{
    if [ "$(type -t pd_set_kernel_panic_triggers)" = "function" ]; then
        pd_set_kernel_panic_triggers
    fi
}

function start()
{
  status 
  rc=$?
  if [ "$rc" == "2" ]; then
    echo -n "Kdump is not supported on this kernel"; failure; echo
    return 1;
    else
      if [ "$rc" == "0" ]; then
        echo -n "Kdump already running"; success; echo
        return 0
      fi
  fi

  check_kernel_parameter
  if [ $? != "0" ]; then
    # crashkernel wasn't defined or equal to 0
    $LOGGER "kexec not started due to crashkernel not defined or equal zero"
    return 0
  fi

  if [ "$(type -t platform_get_dumpdisk_partition)" = "function" ]; then
    dumpdisk=$(platform_get_dumpdisk_partition)
  fi

  if [ -z "$dumpdisk" ]; then
    dumpdisk=$(cat /proc/mounts |  grep `readlink -f /misc/scratch` | awk ' { print $1 } ')
  fi
  if [ -z "$dumpdisk" ]; then
    echo -n "Starting kdump:"; failure; echo
    $LOGGER "Cannot find target partition for kernel dump"
    return 1
  fi

  if [ ! -r $KDUMP_INITRD ]; then
    echo "Pre-built crash initrd not available, triggering runtime creation"
    make_crash_initrd
  fi
  
  load_kdump
  if [ $? != "0" ]; then
    echo -n "Starting kdump:"; failure; echo
    $LOGGER "failed to start up"
    return 1
  fi
  set_kernel_panic_triggers

  echo -n "Starting kdump:"; success; echo
  $LOGGER "started up"
}

function stop()
{
  $KEXEC -p -u 2>/dev/null
  if [ $? == 0 ]; then
    $LOGGER "Unloaded kdump kernel"
    echo -n "Stopping kdump:"; success; echo
    $LOGGER "stopped"
    return 0
  else
    $LOGGER "Failed to unload kdump kernel"
    echo -n "Stopping kdump:"; failure; echo
    $LOGGER "failed to stop"
    return 1
  fi
}

case "$1" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  status)
    EXIT_CODE=0
    status
case "$?" in
    0)
      echo "Kdump is operational"
      EXIT_CODE=0
      ;;
    1)
      echo "Kdump is not operational"
      EXIT_CODE=3
      ;;
    2)
      echo "Kdump is unsupported on this kernel"
      EXIT_CODE=3
      ;;
esac
exit $EXIT_CODE
;;
restart)
     stop
     start
     ;;
condrestart)
     EXIT_CODE=1
     status
case "$?" in
     0)
       stop
       start
       EXIT_CODE=0
       ;;
esac
exit $EXIT_CODE
;;
*)
  echo $"Usage: $0 {start|stop|status|restart}"
  exit 1
esac

exit $?

