#!/usr/bin/env python
# -*- coding:utf8 -*-
# vim: set ts=2 sw=2 sts=2 et:
#
#  Dioivo. A log driven web benchmarking tool.
#  Copyright (C) 2014  Diego Blanco
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, version 3 of the License.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import gzip
import httplib
import numpy as np
import os
import pprint
import sys
import time
import traceback
import urllib3
from argparse import ArgumentParser
from multiprocessing import Process, Event, Queue, Manager
from random import Random
from dioivo_meta.globals import VERSION, AUTHOR, AUTHOR_EMAIL, URL, SOCKET_TIMEOUT, DEFAULT_HEADERS

try:
  if not os.getenv('DISPLAY'):
    import matplotlib as mpl
    mpl.use('Agg')
  import matplotlib.pyplot as plt
except ImportError:
  plt = None


# Disable warnings for invalid certificates
urllib3.disable_warnings( urllib3.exceptions.InsecureRequestWarning )


START_TIME=0
STOP_TIME=0


class dioivoColor():

  def __init__( self ):
    # foreground colors
    self.red     ='\x1b[31m'
    self.lred    ='\x1b[91m'
    self.green   ='\x1b[32m'
    self.lgreen  ='\x1b[92m'
    self.yellow  ='\x1b[33m'
    self.lyellow ='\x1b[93m'
    self.blue    ='\x1b[34m'
    self.lblue   ='\x1b[94m'
    self.magenta ='\x1b[35m'
    self.lmagenta='\x1b[95m'
    self.cyan    ='\x1b[36m'
    self.lcyan   ='\x1b[96m'
    self.grey    ='\x1b[90m'
    self.lgrey   ='\x1b[37m'
    self.white   ='\x1b[97m'
    self.black   ='\x1b[30m'

    # background colors
    self.b_red     ='\x1b[41m'
    self.b_lred    ='\x1b[101m'
    self.b_green   ='\x1b[42m'
    self.b_lgreen  ='\x1b[102m'
    self.b_yellow  ='\x1b[43m'
    self.b_lyellow ='\x1b[103m'
    self.b_blue    ='\x1b[44m'
    self.b_lblue   ='\x1b[104m'
    self.b_magenta ='\x1b[45m'
    self.b_lmagenta='\x1b[105m'
    self.b_cyan    ='\x1b[46m'
    self.b_lcyan   ='\x1b[106m'
    self.b_grey    ='\x1b[100m'
    self.b_lgrey   ='\x1b[47m'
    self.b_white   ='\x1b[107m'
    self.b_black   ='\x1b[40m'

    # special
    self.reset ='\033[0;0m'
    self.bold      ='\033[01m'
    self.italic    ='\033[03m'
    self.underline ='\033[04m'
    self.inverse   ='\033[07m'
    self.conceil   ='\033[08m'
    self.crossedout='\033[09m'
    self.bold_off      ='\033[21m'
    self.italic_off    ='\033[23m'
    self.underline_off ='\033[24m'
    self.inverse_off   ='\033[27m'
    self.conceil_off   ='\033[28m'
    self.crossedout_off='\033[29m'


def print_debug( msg, new_line=True ):
  if new_line:
    print( msg )
  else:
    print( msg ),
  sys.stdout.flush()


class dioivoException( Exception ):
  def __init__(self, msg):
    self.val = msg

  def __str__(self):
    return '\r' + repr( self.val )


class dioivoManager( object ):

  def __init__( self, concurrency ):
    self.pl = []  # Process List
    self.concurrency = concurrency

  def start( self, target, args ):
    try:
      for p in range(self.concurrency):
        p = Process( target=target, args=args )
        self.pl.append( p )
        p.start()

    except KeyboardInterrupt as e:
      print( 'KeyInt OUT' )
    except IOError as e:
      print( 'IOError OUT' )

  def stop( self ):
    # Wait for all proceses to stop
    for p in self.pl:
      p.join(1)
      p.terminate()


class dioivoAgent( object ):

  def __init__( self, host, http_version=11, auth_rule=None, headers={} ):
    self.host = host
    self.http_version = http_version
    self.auth_rule = auth_rule
    self.headers = headers

  def __get_connection_pool( self ):
    if self.http_version == 10:
      httplib.HTTPConnection._http_vsn = 10
      httplib.HTTPConnection._http_vsn_str = 'HTTP/1.0'
    elif self.http_version == 11:
      httplib.HTTPConnection._http_vsn = 11
      httplib.HTTPConnection._http_vsn_str = 'HTTP/1.1'
    else:
      raise dioivoException('Bad HTTP version: %s' % repr(self.http_version))

    if self.auth_rule:
      try:
        auth = self.auth_rule.split('::')
        method=auth[0]
        url=auth[1]

        if url[:7].lower() != 'http://' or url[:8].lower() != 'https://':
          raise dioivoException('Bad authentication rule syntax: URL does not start by http:// or https://')

        params={}
        for p in auth[2:]:
          [k, v] = p.split('=')
          params[k]=v

        tpool = urllib3.connectionpool.connection_from_url(url)
        # Disable certificate verification for auth
        tpool.cert_reqs = 'CERT_NONE'
        tpool.ca_certs = None

        if method.lower() == 'get':
          r = tpool.request('GET', '%s?%s' % (url, '&'.join(auth[2:])))
          cookie = r.getheader('set-cookie')
        elif method.lower() == 'post':
          r = tpool.request('POST', '%s', fields=params)
          cookie = r.getheader('set-cookie')
        else:
          raise dioivoException('Bad authentication rule syntax: Only POST or GET methods allowed.')

        DEFAULT_HEADERS['Cookie'] = cookie

      except IndexError:
        raise dioivoException('Bad authentication rule syntax')

    if self.headers:
      for h in self.headers:
        DEFAULT_HEADERS[ h ] = self.headers[ h ]

    cpool = urllib3.connectionpool.connection_from_url(self.host, headers=DEFAULT_HEADERS)

    # Disable certificate verification for all connections
    cpool.cert_reqs = 'CERT_NONE'
    cpool.ca_certs = None

    return cpool

  def request_worker( self, event, urls, data ):
    """
    event : multiprocessing.Event
    connection_variables: dict
    urls: multiprocessing.Queue
    """
    try:
      event.wait()

      cp = self.__get_connection_pool()

      while not urls.empty():
        try:
          r = None

          # Request
          _url = urls.get()
          start = time.time()
          r = cp.urlopen('GET', _url, retries=0, redirect=False, release_conn=False, preload_content=False, timeout=SOCKET_TIMEOUT)

          # get http code
          code = r.status

          # ttfb (time to first byte)
          r.read(1)
          ttfb = time.time() - start
          # ttlb (time to last byte)
          r.read()
          ttlb = time.time() - start

        except Exception as e:
          if not isinstance(e, (urllib3.exceptions.MaxRetryError)):
            print_debug("\n[%d] Exception occurred: %s\n" % (os.getpid(), e.__repr__()))

        finally:
          # Release connection so the pool can continue doing requests
          # Also get transfered bytes
          if r is not None:
            wire_bytes = r.tell()
            r.release_conn()
          # If r is None then something went wrong and we set code=None
          # so it does not affect timing stats. Also reset the other vars
          else:
            code = None
            ttfb = 0
            ttlb = 0
            wire_bytes = 0

          data.append({
            'time': start,
            'url': _url,
            'code': code,
            'ttfb': ttfb,
            'ttlb': ttlb,
            'bytes': wire_bytes
          })

    except KeyboardInterrupt as e:
      sys.exit( 1 )
    except IOError as e:
      sys.exit( 1 )
    except EOFError as e:
      sys.exit( 1 )


class dioivoUrlProvider( object ):

  def __init__(self, urls_file_path, mode, requests_number, replace_parameter_list=[], methods=['GET']):
    """mode = [random,squence]"""

    if mode.lower() not in ('random', 'sequence'):
      raise dioivoException('Invalid url choosing mode')

    self.requests_number = requests_number
    self.mode = mode.lower()
    self.replace_parameter_dict = {}
    self.__parse_replace_parameter( replace_parameter_list )
    self.methods = methods

    try:
      self.queue = self.__filter_urls(urls_file_path)
    except IOError as e:
      raise dioivoException('Unable to open file: %s. %s' % (urls_file_path, e.__str__()))

  def __parse_replace_parameter(self, rpl):
    for rp in rpl:
      npk, npv = rp.split('=')
      self.replace_parameter_dict[npk] = npv

  def __filter_urls(self, access_file):
    urls = []
    urls_to_use = Queue()
    r = Random()

    if access_file.lower()[-3:] == '.gz':
      open_function = gzip.open
    else:
      open_function = open

    with open_function(access_file) as access_lines:
      for line in access_lines:
        fields = line.split()
        for m in self.methods:
          try:
            M = m.upper()
            if M not in fields:
              # In access.log files methods are preceded by double quotes
              M = '"'+M
            u = fields[fields.index( M ) + 1]
            urls.append( u )
          finally:
            break

    url_index = None
    n_urls = urls.__len__()
    if n_urls > 0:
      for i in range(0, self.requests_number):
        if self.mode == 'random':
          u = urls[ r.randint(0, n_urls-1) ]
        elif self.mode == 'sequence':
          # (Re)set the index
          if url_index is None or url_index == n_urls-1:
            url_index = -1
          url_index+=1
          u = urls[ url_index ]

        url = u.strip()

        # If url is not a relative path, search for other url
        if url[:7].lower() == "http://" or url[:8].lower() == "https://":
          i+=1
          continue

        if self.replace_parameter_dict != {} and url.find('?') != -1:
          [rawurl, rawp] = url.split('?')
          params='?'
          for p in rawp.split('&'):
            [k, v]=p.split('=')
            if k in self.replace_parameter_dict:
              params+="%s=%s&" % (k, self.replace_parameter_dict[k])
            else:
              params+="%s=%s&" % (k, v)
          # remove last &
          params=params[:-1]
          url = rawurl+params
        urls_to_use.put( url )
    else:
      raise dioivoException( 'No urls found for methods: %s' % repr( self.methods ) )

    return urls_to_use


class dioivoDataManager( object ):

  def __init__(self, total_requests, concurrency, http_version):
    self.data = {}

    self.total_requests = total_requests
    self.concurrency = concurrency
    self.http_version = http_version

    self.plot_ttfb_data = []
    self.plot_ttlb_data = []
    self.plot_time_data = []
    self.stat = None
    self.output = ''

  def add_data(self, d):
    self.data[d['time']] = d
    self.plot_ttfb_data.append( d['ttfb'] )
    self.plot_ttlb_data.append( d['ttlb'] )
    self.plot_time_data.append( d['time'] - START_TIME )

  def save(self, filenames_prefix='output'):
    with open(filenames_prefix+'.csv', 'w') as f:
      dkeys = self.data.keys()
      dkeys.sort()
      f.write('time\t\turl\t\tttfb\t\tttlb\n')
      for k in dkeys:
        f.write('%f\t\t%s\t\t%f\t\t%f\n' % ( self.data[k]['time'], self.data[k]['url'], self.data[k]['ttfb'], self.data[k]['ttlb']))

    with open(filenames_prefix+'.stat', 'w') as f:
      pprint.pprint(self.stat, stream=f)

    with open(filenames_prefix+'.txt', 'w') as f:
      f.write(self.output)

    if plt:
      self.replot()
      plt.savefig('%s.png' % filenames_prefix, dpi=300)

  def compile_stats(self):
    self.stat={
      'ttfb': {
          'min': None,
          'avg': None,
          'max': None
      },
      'ttlb': {
          'min': None,
          'avg': None,
          'max': None
      },
      'mbps': {
          'min': None,
          'avg': None,
          'max': None
      },
      'url': {
          'min_ttfb': '',
          'max_ttfb': '',
          'min_ttlb': '',
          'max_ttlb': ''
      },
      'code': {
      },
      'rps': 0
    }

    # Stats for TTFB and TTLB
    count=0
    for d in self.data:
      # Init codes
      if self.data[d]['code'] in self.stat['code']:
        self.stat['code'][self.data[d]['code']]+=1
      else:
        self.stat['code'][self.data[d]['code']]=1

      # Avoid failed request to affect stats
      if self.data[d]['code'] is None:
        continue

      # Calculate bandwidth for this request in Mbps
      mbps = ( self.data[d]['bytes'] * 8.0 / (self.data[d]['ttlb']-self.data[d]['ttfb']) ) / 1024**2

      count+=1
      # Init
      if self.stat['ttfb']['min'] is None:
        self.stat['ttfb']['min'] = self.data[d]['ttfb']
        self.stat['ttfb']['avg'] = self.data[d]['ttfb']
        self.stat['ttfb']['max'] = self.data[d]['ttfb']
        self.stat['ttlb']['min'] = self.data[d]['ttlb']
        self.stat['ttlb']['avg'] = self.data[d]['ttlb']
        self.stat['ttlb']['max'] = self.data[d]['ttlb']
        self.stat['mbps']['min'] = mbps
        self.stat['mbps']['avg'] = mbps
        self.stat['mbps']['max'] = mbps
      else:
        # Min
        if self.data[d]['ttfb'] < self.stat['ttfb']['min']:
          self.stat['ttfb']['min'] = self.data[d]['ttfb']
        if self.data[d]['ttlb'] < self.stat['ttlb']['min']:
          self.stat['ttlb']['min'] = self.data[d]['ttlb']
        if mbps < self.stat['mbps']['min']:
          self.stat['mbps']['min'] = mbps

        # Max
        if self.data[d]['ttfb'] > self.stat['ttfb']['max']:
          self.stat['ttfb']['max'] = self.data[d]['ttfb']
        if self.data[d]['ttlb'] > self.stat['ttlb']['max']:
          self.stat['ttlb']['max'] = self.data[d]['ttlb']
        if mbps > self.stat['mbps']['max']:
          self.stat['mbps']['max'] = mbps

        # Avg
        self.stat['ttfb']['avg'] = self.stat['ttfb']['avg'] + (self.data[d]['ttfb']-self.stat['ttfb']['avg'])/float(count)
        self.stat['ttlb']['avg'] = self.stat['ttlb']['avg'] + (self.data[d]['ttlb']-self.stat['ttlb']['avg'])/float(count)
        self.stat['mbps']['avg'] = self.stat['mbps']['avg'] + (mbps-self.stat['mbps']['avg'])/float(count)

    req_count=count

    # self.stats for URL
    ud = {}
    for d in self.data:
      if self.data[d]['url'] not in ud:
        # Init ud for new url with ttfb, ttlb and the count of url occurrences (for mean calculation)
        ud[self.data[d]['url']]=[ self.data[d]['ttfb'], self.data[d]['ttlb'], 1]
      else:
        # Calculate mean values for ttfb and ttlb for this url
        ud[self.data[d]['url']][2] += 1
        count = ud[self.data[d]['url']][2]
        # 0: ttfb mean for this url
        ud[self.data[d]['url']][0] = ud[self.data[d]['url']][0] +  (self.data[d]['ttfb'] - ud[self.data[d]['url']][0])/count
        # 1: ttlb mean for this url
        ud[self.data[d]['url']][1] = ud[self.data[d]['url']][1] +  (self.data[d]['ttlb'] - ud[self.data[d]['url']][1])/count

    for k in ud:
      # Init
      if not self.stat['url']['min_ttfb']:
        self.stat['url']['min_ttfb'] = (ud[k][0], k)
        self.stat['url']['max_ttfb'] = (ud[k][0], k)
        self.stat['url']['min_ttlb'] = (ud[k][1], k)
        self.stat['url']['max_ttlb'] = (ud[k][1], k)
      else:
        if ud[k][0] < self.stat['url']['min_ttfb'][0]:
          self.stat['url']['min_ttfb']=(ud[k][0], k)
        if ud[k][0] > self.stat['url']['max_ttfb'][0]:
          self.stat['url']['max_ttfb']=(ud[k][0], k)
        if ud[k][1] < self.stat['url']['min_ttlb'][0]:
          self.stat['url']['min_ttlb']=(ud[k][1], k)
        if ud[k][1] > self.stat['url']['max_ttlb'][0]:
          self.stat['url']['max_ttlb']=(ud[k][1], k)

    # get requests per second
    self.stat['rps']=req_count/( STOP_TIME-START_TIME )

    return self.stat

  def print_stats(self):
    if self.stat['ttfb']['min'] is None:
      raise dioivoException( 'All requests failed' )
    self.output+= ' Stats\n\n'
    self.output+= ' \tminimum\t\taverage\t\tmaximum\n'
    self.output+= ' \t-------\t\t-------\t\t-------\n'
    self.output+= ' ttfb\t%.5f s\t%.5f s\t%.5f s\n' % (self.stat['ttfb']['min'], self.stat['ttfb']['avg'], self.stat['ttfb']['max'])
    self.output+= ' ttlb\t%.5f s\t%.5f s\t%.5f s\n\n' % (self.stat['ttlb']['min'], self.stat['ttlb']['avg'], self.stat['ttlb']['max'])
    self.output+= ' mbps\t%.3f Mbps\t%.3f Mbps\t%.3f Mbps\n' % (self.stat['mbps']['min'], self.stat['mbps']['avg'], self.stat['mbps']['max'])
    self.output+= '\n\n'
    self.output+= ' Requests per second: %d rps\n' % self.stat['rps']
    self.output+= '\n\n'
    self.output+= ' URL min ttfb: (%.5f s) %s\n' % self.stat['url']['min_ttfb']
    self.output+= ' URL max ttfb: (%.5f s) %s\n' % self.stat['url']['max_ttfb']
    self.output+= ' URL min ttlb: (%.5f s) %s\n' % self.stat['url']['min_ttlb']
    self.output+= ' URL max ttlb: (%.5f s) %s\n' % self.stat['url']['max_ttlb']
    self.output+= ' NOTE: These stats are based on the average time (ttfb or ttlb) for each url.\n'
    self.output+= '\n\n'
    self.output+= ' Protocol stats:\n'
    code_list = self.stat['code'].keys()
    code_list.sort()
    for c in code_list:
      # Avoid None codes due to connection errors
      if c:
        self.output+= '\tHTTP %d: ' % c
      else:
        self.output+= '\tURL fail: '
      self.output+= '%5d requests (%5.2f%%)\n' % (self.stat['code'][c], 100*float(self.stat['code'][c])/self.total_requests)
    print( self.output )

  def replot(self):
    if plt:
      order = np.argsort( self.plot_time_data )
      xs = np.array( self.plot_time_data )[order]
      ttfb_ys = np.array( self.plot_ttfb_data )[order]
      ttlb_ys = np.array( self.plot_ttlb_data )[order]

      plt.title('Requests: %d   User concurrency: %d   HTTP %.1f' % (self.total_requests, self.concurrency, self.http_version/10.), loc='right')
      plt.xlabel('benchmark time (s)')
      plt.ylabel('request time (s)')
      plt.plot( xs, ttfb_ys, color="green", linewidth=1.0, linestyle="-", label="ttfb" )
      plt.plot( xs, ttlb_ys, color="blue", linewidth=1.0, linestyle="-", label="ttlb" )
      plt.legend()


def dioivo_getopts():
  parser = ArgumentParser(usage="usage: %(prog)s [options] -s SERVER -u URLS_FILE -c CONCURRENCY -n NUMBER_OF_REQUESTS")

  parser.add_argument("-i", "--test-id",
                      action="store",
                      dest="test_id",
                      type=str,
                      default=False,
                      help="Identificator for the test. This name will be a suffix for the output files.")

  parser.add_argument("-s", "--server",
                      action="store",
                      dest="host",
                      type=str,
                      default=None,
                      help="Host to benchmark. It must include the protocol and lack of trailing slash. For example: https://example.com")

  parser.add_argument("-u", "--urls",
                      action="store",
                      dest="urls_file",
                      type=str,
                      default=False,
                      help="File with url's to test. This file must be directly an access.log file from nginx or apache. It can be gzipped (must end with .gz).")

  parser.add_argument("-c", "--concurrency",
                      action="store",
                      dest="concurrency",
                      type=int,
                      default=False,
                      help="Number of concurrent requests")

  parser.add_argument("-n", "--number-of-requests",
                      action="store",
                      dest="total_requests",
                      type=int,
                      default=False,
                      help="Number of requests to send to the host.")

  parser.add_argument("-m", "--mode",
                      action="store",
                      dest="mode",
                      type=str,
                      default='random',
                      help="Mode can be 'random' or 'sequence'. It defines how the urls will be chosen from the url's file.")

  parser.add_argument("-R", "--replace-parameter",
                      action="append",
                      dest="replace_parameter",
                      type=str,
                      default=[],
                      help="Replace parameter on the URLs that have such parameter: p.e.: 'user=hackme' will set the parameter 'user' to 'hackme' on all url that have the 'user' parameter. Can be called several times to make multiple replacements.")

  parser.add_argument("-A", "--auth",
                      action="store",
                      dest="auth_rule",
                      type=str,
                      default=None,
                      help="Adds rule for form authentication with cookies. Syntax: 'METHOD::URL[::param1=value1[::param2=value2]...]'. For example: 'POST::http://example.com/login.py::user=root::pass=hackme'. NOTE: Use full url with protocol as in the example.")

  parser.add_argument("--http-version",
                      action="store",
                      dest="http_version",
                      type=int,
                      default='11',
                      help="Defines which protocol version to use. Use '11' for HTTP 1.1 and '10' for HTTP 1.0")

  parser.add_argument("-H", "--add-header",
                      action="append",
                      dest="headers",
                      type=lambda kv: kv.split(":", 1),
                      help="Adds a header to the requests. It can be specified multiple times. For example: -H 'User-Agent: MyUserAgent'")

  parser.add_argument("-e", "--examine",
                      action="store_true",
                      dest="examine_plot",
                      default=False,
                      help="If you enable this flag, you'll be able to examine the graph for the benchmark when it finishes. Note that this requires matplotlib and an X environment.")

  parser.add_argument("-M", "--methods",
                      action="store",
                      dest="methods",
                      type=lambda v: [ item for item in v.split(',') ],
                      default=['GET'],
                      help="Comma separated list of methods to be looked for in the access log file. Default only GET requests are used. For example: 'GET,PUT,DELETE'")

  parser.add_argument('-D', '--debug',
                      action='store_true',
                      dest="debug",
                      default=False,
                      help='Show debug information. For debugging only.')

  parser.add_argument('-V', '--version',
                      action='store_true',
                      dest="version",
                      default=False,
                      help='Show debug information. For debugging only.')

  opts = parser.parse_args()

  if opts.version:
    print( "dioivo %s\nAuthor: %s <%s>\nProject: %s" % (
      VERSION,
      AUTHOR,
      AUTHOR_EMAIL,
      URL))
    sys.exit( 0 )

  if not opts.host:
    parser.error('Missing host to test (-s)')
  if not opts.urls_file:
    parser.error('Missing file with urls (-u)')
  if not opts.concurrency:
    parser.error('Concurrency not given (-c)')
  if not opts.total_requests:
    parser.error('Number of requests not given (-n)')
  if opts.examine_plot and not os.getenv('DISPLAY'):
    parser.error('X Server not available for live display (-l)')

  try:
    if opts.headers:
      opts.headers = dict(opts.headers)
  except Exception as e:
    parser.error('Wrong header format (-H)')

  return opts


def dioivo_main( opts ):
  if not plt:
    print( '\nWARNING: Could not import matplotlib. Graphs will not be generated\n' )

  print( '\nSetting up...' ),
  sys.stdout.flush()

  # Colors
  C = dioivoColor()

  # Create needed instances
  dup = dioivoUrlProvider(
    opts.urls_file,
    opts.mode,
    opts.total_requests,
    replace_parameter_list=opts.replace_parameter,
    methods = opts.methods
  )
  ddm = dioivoDataManager( opts.total_requests, opts.concurrency, opts.http_version )
  dag = dioivoAgent( opts.host, opts.http_version, opts.auth_rule, opts.headers )

  raw_data = Manager().list()
  event = Event()

  # Create processes for concurrency
  dm = dioivoManager( opts.concurrency )
  dm.start(
    target = dag.request_worker,
    args = ( event, dup.queue, raw_data )
  )

  # Set event so we ensure all threads
  # start the benchmark at the same time.
  global START_TIME
  START_TIME=time.time()
  event.set()

  print( "\r Requests: %d\t\tConcurrency: %d" % ( opts.total_requests, dm.pl.__len__()) )
  # bs: progress bar size
  bs=50

  # Main loop with progress bar and plotting
  time.sleep(1)
  last_qsize = dup.queue.qsize()
  while not dup.queue.empty():
    qsize = dup.queue.qsize()
    per=(qsize*bs)/opts.total_requests
    print( ' %s%s%s%s%s %d rps     \r' % (C.crossedout+C.white, ' '*(bs-per-1), C.grey, ' '*per, C.reset, (last_qsize-qsize)*2) ),
    sys.stdout.flush()
    time.sleep(0.5)
    last_qsize = qsize
    sys.stdout.flush()
  print( '%s              \n' % (' '*bs) ),
  sys.stdout.flush()
  global STOP_TIME
  STOP_TIME=time.time()

  print( ' Stopping...' ),
  sys.stdout.flush()
  dm.stop()

  print( "\r Compiling stats..." ),
  sys.stdout.flush()
  for d in raw_data:
    ddm.add_data(d)
  ddm.compile_stats()

  print( "\r                    \r" ),
  sys.stdout.flush()
  ddm.print_stats()

  # Saving information
  file_prefix='%s_r%d_c%d' %(opts.host.split('://')[1], opts.total_requests, opts.concurrency)
  if opts.test_id:
    file_prefix='%s_%s' %(opts.test_id, file_prefix)

  ddm.save( file_prefix )

  if opts.examine_plot and plt:
    plt.show()


if __name__ == '__main__':
  try:
    opts = dioivo_getopts()
    dioivo_main( opts )
  except dioivoException as e:
    print( 'Error: %s' % e.__str__() )
    sys.exit(1)
  except KeyboardInterrupt as e:  # Ctrl-C
    sys.exit( 130 )
  except SystemExit as e:  # sys.exit()
    raise e
  except Exception as e:
    if opts.debug:
      traceback.print_exc()
    else:
      print( str(e) )
    os._exit(1)
