#!/usr/bin/env bash
#
# Setup Python Virtual Environment for Qualys API ETL Example Code
# This code is called after initial installation on base operating system
# to create the python virtual environment and install the Qualys API ETL Example Code
# into the python virtual environment.
#
# Last Updated with Version: 0.7.52
#
# Options Examples to install QualysETL.
# All supporting software is installed as latest from pypi.org.
#
# Install latest from pypi.org        ~/.local/bin/qetl_setup_python_venv
# Install latest from pypi.org        ~/.local/bin/qetl_setup_python_venv /opt/qetl
# Install latest from test.pypi.org   ~/.local/bin/qetl_setup_python_venv /opt/qetl test
# Install version from test.pypi.org  ~/.local/bin/qetl_setup_python_venv /opt/qetl test 0.6.131
# Install version from pypi.org       ~/.local/bin/qetl_setup_python_venv /opt/qetl prod 0.6.131

export PATH=$PATH:/bin:/usr/bin
export default_home="/opt/qetl"
export default_pypi="prod"
export default_pypi_version="latest"
export print_help_flag="no"
export qetl_print_help_flag="${1:-$print_help_flag}"
export qetl_HOME="${1:-$default_home}" # default /opt/qetl
export qetl_default_pypi="${2:-$default_pypi}"
export qetl_default_pypi_version="${3:-$default_pypi_version}" # default prod pypi.org
export LANG='en_US.UTF-8'
export PYTHONUNBUFFERED="1"

function print_help_instructions_qetl_setup_python_env() {
    cat <<-EOF

    usage:       qetl_setup_python_venv [/opt/qetl] [test|prod] [version number]
                 qetl_setup_python_venv [-h] for help

    description:

        Create a python3 virtual environment and install the qualysetl
        application into that environment for usage.  This isolates the
        qualysetl application dependencies to the python3 virtual environment.
        See https://pypi.org/project/qualysetl/ for first time setup and
        installation instructions.

    options:

        qetl_setup_python_venv [/opt/qetl] [test|prod] [version number]

        1) [/opt/qetl]        - root directory where application and data
                                will be stored.
                              - You must be root to create this directory.
                              - See https://pypi.org/project/qualysetl/ for
                                first time setup/installation instructions.
        2) [test|prod]        - obtain QualysETL from test or prod pypi
                                instance.
        3) [version number]   - obtain version number of qualysetl.


    examples:
            1) qetl_setup_python_venv /opt/qetl
               - Ensure you have /opt/qetl directory created before running
                 this program.
               - Creates QualysETL Environment.  See directory information
                 below.

            2) qetl_setup_python_venv /opt/qetl prod 0.6.131
               - will install version 0.6.131 of qualysetl from pypi.org into
                 your /opt/qetl/qetl_venv directory.

            3) qetl_setup_python_venv /opt/qetl test 0.6.131
               - will install version 0.6.131 of qualysetl from test.pypi.org
                 into your /opt/qetl/qetl_venv directory.

    directory information:
             /opt/qetl            - root directory for Application and Data
             /opt/qetl/qetl_venv  - application directory for Qualys ETL
                                    Python Virtual Environment
             /opt/qetl/users      - data directory containing results of
                                    QualysETL execution.

    files:
        See https://dg-cafe.github.io/qualysetl/#application-manager-and-data


    container notes:

            1) For container deployment, ex docker, application and data
               are separated for container deployment.

               Container Application - /opt/qetl/qetl_venv should installed into the container image.
               Persistent Data       - /opt/qetl/users should be mapped to the underlying host
                                       system for persistent storage of application data.


EOF
}
function qetl_manage_user_code() {
    cat <<-EOF
#!/usr/bin/env python3
# DO NOT EDIT MANUALLY.  GENERATED PROGRAM on $(date)
# qetl_manage_user is generated by qetl_setup_python_venv
#
# -*- encoding: utf-8 -*-
import re
import sys
import os
from pathlib import Path

# These variables are set by qap_setup_python_env upon installation.
global opt_qetl_dir     # /opt/qetl or $SOMEDIR/opt/qetl
global virtual_env_dir
global virtual_env_bin_python3
global virtual_env_bin_activation
global virtual_env_this_program


def error_out(error_message):
    print("")
    print(f"Error: {error_message}")
    print("")
    print(f"Please activate your python virtual environment before executing this script.")
    print(f"  Your python venv is: {virtual_env_dir}")
    print("")
    print("   Enter your python venv before starting this program:")
    print(f"    1) source {virtual_env_bin_activation}")
    print(f"    2) {virtual_env_this_program}")
    print("")
    print("Retry when ready.")
    exit(1)


def validate_python_venv_is_correct():
    global opt_qetl_dir
    global virtual_env_dir
    global virtual_env_bin_python3
    global virtual_env_bin_activation
    global virtual_env_this_program

    # These variables are set by qap_setup_python_env upon installation.
    opt_qetl_dir = '$qetl_BASE'
    os.environ['OPT_qetl_DIR'] = opt_qetl_dir
    virtual_env_dir = '$PYTHON_VENV'
    virtual_env_bin_python3 = '$PYTHON_VENV_BIN_PYTHON3'
    virtual_env_bin_activation = '$PYTHON_VENV_BIN_ACTIVATE_PYTHON_ENV'
    virtual_env_this_program = '$PYTHON_VENV_BIN_qetl_MANAGE_USER'

    if os.environ.keys().__contains__("VIRTUAL_ENV"):
        pass
    else:
        error_out("VIRTUAL_ENV variable not found..")

    this_script_path = Path(__file__).absolute()
    if str(this_script_path) == virtual_env_this_program:
        pass
    else:
        error_out(f"Relocation error.  Paths are not equal\n"
                  f"    This program path = {this_script_path} \n"
                  f"    Original program path = {virtual_env_this_program}")


from qualys_etl.qetl_manage_user import main
if __name__ == '__main__':
    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
    validate_python_venv_is_correct()
    sys.exit(main())

EOF
}
function print_help_instructions_for_qetl_manage_user() {
cat <<-EOF

   Success! Your python virtual environment for qetl is: $PYTHON_VENV

   Your python3 venv separates your base python installation from the qetl python requirements
   and is your entry to executing the qetl_manage_user application.  Your base qetl installation has
   moved to your python virtual environment: $PYTHON_VENV

   !!! save these commands as they are your entry to run the qetl application

       1) source ${PYTHON_VENV}/bin/activate
       2) ${PYTHON_VENV}/bin/qetl_manage_user  ( Your entry point to operating qualysetl )

   Next steps:

    Enter your python3 virtual environment and begin testing qualys connectivity.

       1) source ${PYTHON_VENV}/bin/activate
       2) ${PYTHON_VENV}/bin/qetl_manage_user

EOF
}
function check_return() {
    if [[ $1 == 0 ]]
       then
       :
    else
       echo "$2"
       exit 1
    fi
}
function start_message() {
    echo "Start $(basename $0) - $(date)"
}
function end_message() {
    echo "End   $(basename $0) - $(date)"
}
function test_os_for_required_commands() {
    echo "  1) $FUNCNAME"
    python3 --version >/dev/null 2>&1
    check_return $? "Problem finding python3.  please install python3 > 3.8 to continue."
    python3 -m pip list >/dev/null 2>&1
    check_return $? "Problem executing python3 -m pip list.  please resolve python3 pip issue and continue."
    basename "/" >/dev/null 2>&1
    check_return $? "Problem finding 'basename', please install gnu tools basename to continue."
    realpath "/" >/dev/null 2>&1
    check_return $? "Problem finding 'realpath', please install gnu tools realpath to continue."
    which yes >/dev/null 2>&1
    check_return $? "Problem finding 'yes' command, please install gnu tools 'yes' to continue."
}

function test_for_pip_connectivity() {
    echo "  2) $FUNCNAME"
    python3 -m pip list >/dev/null 2>&1
    check_return $? "Problem executing python3 -m pip list.  please resolve python3 pip issue and continue."
}
function setup_venv_qetl_base_app_directories() {
      # remove .. remove //  remove /$  remove qetl/home
    export qetl_TEST_BASE_PATH=$(echo "/$qetl_HOME" | sed 's/\.\.//g' | sed 's?//?/?g' | sed 's?/$??g' | sed 's?opt/qetl??g')
    export qetl_BASE="${qetl_TEST_BASE_PATH}/opt/qetl"
    export qetl_BASE=$(echo "$qetl_BASE" | sed 's?//?/?'g)
    export qetl_BASE=$(realpath -L -s $qetl_BASE)
}

function test_venv_qetl_base_app_is_writable() {
    if [[ "$qetl_TEST_BASE_PATH" == "/" ]]; then
        # Check /opt/qetl is created and writable by user
        if [[ ( ( -d "$qetl_BASE" ) && ( -w "$qetl_BASE" ) ) ]]; then
            :  # opt/qetl is a directory and writable
        else
            echo "Please ensure /opt/qetl directory is created and is writable. "
            echo "  - /opt/qetl requires root authorization to create directory and set permissions to writable."
            echo "You selected: $qetl_HOME, which locates the app user to /opt/qetl. "
            echo "Please rerun program when /opt/qetl is writable by user $(logname) or select another directory"
            exit 1
        fi
    elif [[ ( ( -d "$qetl_TEST_BASE_PATH" ) && ( -w "$qetl_TEST_BASE_PATH" ) ) ]]; then
        :
    else
         echo
         echo "Please pre-create a directory that is writable.  You selected: $qetl_HOME, exit program"
         print_help_instructions_qetl_setup_python_env
         exit 1
    fi
}

function setup_python_venv_bin_vars() {
    export PYTHON_VENV="${qetl_BASE}/qetl_venv"
    export PYTHON_VENV_BIN="${qetl_BASE}/qetl_venv/bin"
    export PYTHON_VENV_BIN_PYTHON3="${qetl_BASE}/qetl_venv/bin/python3"
    export PYTHON_VENV_BIN_qetl_MANAGE_USER="${qetl_BASE}/qetl_venv/bin/qetl_manage_user"
    export PYTHON_VENV_BIN_ACTIVATE_PYTHON_ENV="${qetl_BASE}/qetl_venv/bin/activate"
    export REMOVE_qetl_SETUP_PYTHON_VENV_FROM_NEW_VENV="${qetl_BASE}/qetl_venv/bin/qetl_setup_python_venv"
}

# KEEP FOR REFERENCE function remove_old_python_venv() {
# KEEP FOR REFERENCE     local current_directory=$(pwd)
# KEEP FOR REFERENCE     if [[ ( -d "$PYTHON_VENV" ) ]]; then
# KEEP FOR REFERENCE        cd $PYTHON_VENV
# KEEP FOR REFERENCE        check_return $? "Problem changing to directory $PYTHON_VENV, determine why you cd to this directory. "
# KEEP FOR REFERENCE        cd ..
# KEEP FOR REFERENCE        check_return $? "Problem changing to parent directory of $PYTHON_VENV, determine why you cd to this directory. "
# KEEP FOR REFERENCE        /bin/rm -r qetl_venv
# KEEP FOR REFERENCE        check_return $? "Problem removing directory $PYTHON_VENV, determine why the delete directory failed. "
# KEEP FOR REFERENCE    fi
# KEEP FOR REFERENCE    cd $current_directory
# KEEP FOR REFERENCE
# KEEP FOR REFERENCE }

function prepare_opt_qetl_env_dirs() {
    echo "  3) $FUNCNAME"
    setup_venv_qetl_base_app_directories
    test_venv_qetl_base_app_is_writable
    setup_python_venv_bin_vars
}
function test_ask_user_if_they_want_to_create_python_venv() {

    print_help_instructions_qetl_setup_python_env
    echo
    echo "Create qetl Python Environment? $PYTHON_VENV $qetl_default_pypi:$qetl_default_pypi_version"
    while true; do
        read -p "Do you want to create your python3 virtual environment for qetl? ( yes or no ) " yn
        case $yn in
            [yY]* ) echo;echo "ok, creating python3 virtual $PYTHON_VENV $qetl_default_pypi:$qetl_default_pypi_version";echo; break;;
            [Nn]* ) echo;echo "Thank you, exiting";echo; exit;;
            * ) echo "Please answer yes or no.";;
        esac
    done
}

function install_qetl() {
   if [[ "$qetl_default_pypi" == "test" ]]; then
       if [[ "$qetl_default_pypi_version" == "latest" ]]; then
           python3 -m pip install --upgrade -i https://test.pypi.org/simple/ qualysetl

           check_return $? "Sorry, problem installing qualysetl from test.pypi.org. resolve and retry."
           echo "     -  installed latest qualysetl from https://test.pypi.org/"
       else
           python3 -m pip install -i https://test.pypi.org/simple/ qualysetl==$qetl_default_pypi_version
           check_return $? "Sorry, problem installing qualysetl version $qetl_default_pypi_version from test.pypi.org.  resolve and retry."
	   echo "     - installed qualysetl version $qetl_default_pypi_version from https://test.pypi.org/"
       fi
   else
       if [[ "$qetl_default_pypi_version" == "latest" ]]; then
           python3 -m pip install --upgrade qualysetl >/dev/null 2>&1
           check_return $? "Sorry, problem installing qualysetl from pypi.org. resolve and retry."
	   echo "     - installed latest qualysetl from https://pypi.org/"
       else
           python3 -m pip install qualysetl==$qetl_default_pypi_version >/dev/null 2>&1
           check_return $? "Sorry, problem installing qualysetl version $qetl_default_pypi_version from pypi.org.  resolve and retry."
	   echo "     - installed qualysetl version $qetl_default_pypi_version from https://pypi.org/"
      fi
   fi
}

function create_qetl_python_venv() {
    echo
    echo "  4) $FUNCNAME - will run for about 1-2 minutes"
    echo
    mkdir -p $qetl_BASE >/dev/null 2>&1
    if [[ -d ${qetl_BASE} ]]; then
       python3 -m venv $PYTHON_VENV
       check_return $? "Problem creating python virtual environment $PYTHON_VENV"

       qetl_manage_user_code > "$PYTHON_VENV_BIN_qetl_MANAGE_USER"
       check_return $? "Problem creating $PYTHON_VENV_BIN_qetl_MANAGE_USER"

       chmod 755 "$PYTHON_VENV_BIN_qetl_MANAGE_USER"
       check_return $? "Problem setting permissions on $PYTHON_VENV_BIN_qetl_MANAGE_USER"

       source ${PYTHON_VENV}/bin/activate
       check_return $? "Problem with source python virtual environment $PYTHON_VENV"

# Added Upgrade to pip in venv starting with 0.7.52

       python3 -m pip install --upgrade pip >/dev/null
       check_return $? "Sorry, problem installing pip package into python virtual env $PYTHON_VENV, resolve and retry."

       python3 -m pip install --upgrade setuptools wheel >/dev/null
       check_return $? "Sorry, problem installing setuptools/wheel packages into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade requests >/dev/null
       check_return $? "Sorry, problem installing requests package into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade chardet >/dev/null
       check_return $? "Sorry, problem installing chardet package into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade oschmod >/dev/null
       check_return $? "Sorry, problem installing oschmod package into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade pyyaml >/dev/null
       check_return $? "Sorry, problem installing pyyaml package into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade xmltodict >/dev/null
       check_return $? "Sorry, problem installing xmltodict package into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade boto3 >/dev/null
       check_return $? "Sorry, problem installing boto3 base package into python virtual env $PYTHON_VENV, resolve and retry."

# Added psutil in 0.7.51 to support capturing system utilization in logging.
       python3 -m pip install --upgrade psutil >/dev/null
       check_return $? "Sorry, problem installing psutil base package into python virtual env $PYTHON_VENV, resolve and retry."

# Added lxml, matplotlib, pandas, pyarrow with 0.7.52 to support parquet and other pandas/pyarrow conversions.

       python3 -m pip install --upgrade lxml >/dev/null
       check_return $? "Sorry, problem installing lxml base package into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade matplotlib >/dev/null
       check_return $? "Sorry, problem installing matplotlib base package into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade pandas >/dev/null
       check_return $? "Sorry, problem installing pandas base package into python virtual env $PYTHON_VENV, resolve and retry."
       python3 -m pip install --upgrade pyarrow >/dev/null
       check_return $? "Sorry, problem installing pyarrow base package into python virtual env $PYTHON_VENV, resolve and retry."

       install_qetl
       check_return $? "Sorry, problem installing base packages into python virtual env $PYTHON_VENV, resolve and retry."

       python3 -m pip list | sed 's/^/    /' | nl
       check_return $? "Sorry, problem using pip to list new env packages in env: $PYTHON_VENV, resolve and retry."

       echo;echo
       python3 -m pip show qualysetl  | sed 's/^/    /' | nl
       /bin/rm "${qetl_BASE}/qetl_venv/bin/qetl_setup_python_venv" >/dev/null 2>&1
    else
       echo "Problem creating $qetl_BASE directory.  Check permissions and rerun."
       print_qetl_setup_python_env_help_instructions
       exit 1
    fi
}

function test_for_help_message() {

   if [[ "$qetl_print_help_flag" == "-h" ]]; then
      print_help_instructions_qetl_setup_python_env
      exit 0
   elif [[ "$qetl_print_help_flag" == "--help" ]]; then
      print_help_instructions_qetl_setup_python_env
      exit 0
   fi

}

function if_root_user_exit_program() {
  if [[ $EUID -eq 0 ]]; then
     echo "This program cannot be run as root.  Please run as a non-root user for QualysETL to function properly."
     exit 1
  fi

}

#
# MAIN
#
if_root_user_exit_program
start_message
test_for_help_message
test_os_for_required_commands
test_for_pip_connectivity
prepare_opt_qetl_env_dirs
test_ask_user_if_they_want_to_create_python_venv
create_qetl_python_venv
print_help_instructions_for_qetl_manage_user
end_message
