Install update

hammerdb/hammerdb

@@ -23,6 +23,10 @@ log_mount=""
 pcp=""
 pdir=""
 script_dir=$(realpath $(dirname $0))
+export script_dir
+hmd_dir=${script_dir}/hammerdb-tpcc/Hammerdb
+mkdir -p $hmd_dir
+export hmd_dir
 
 curdir=`pwd`
 if [[ $0 == "./"* ]]; then
@@ -129,6 +133,7 @@ usage()
 
 ${TOOLS_BIN}/gather_data ${curdir}
 source ${TOOLS_BIN}/general_setup "$@"
+export to_pkg_tool_flags
 export to_no_pkg_install
 
 unset DISPLAY
@@ -289,7 +294,7 @@ if [[ $to_tuned_setting != "none" ]]; then
 	tuned_original=`tuned-adm active | cut -d' ' -f4`
 	tuned-adm profile $to_tuned_setting
 fi
-pushd /usr/local > /dev/null
+pushd $hmd_dir > /dev/null
 rm -rf $test
 popd > /dev/null
 cpdir=""
@@ -303,25 +308,16 @@ echo ./run_hammerdb -m /perf1 -t ${test} ${users_to_run} ${warehouses} ${log_mou
 chmod 755 run_this
 ./run_this
 
-cd /usr/local
-mv HammerDB $test
-
-${TOOLS_BIN}/move_data $curdir $test
-pushd $test
-files=`ls test*out`
-if [ $? -ne 0 ]; then
-	echo Failed >> test_results_report
-else
-	echo Ran >> test_results_report
-fi
-
+#pwd
+#echo $hmd_dir $test
+rm -rf $test
+mv $hmd_dir $test
 cp /tmp/hammerdb.out .
-popd
 
 tmp_file=`mktemp /tmp/hammer_data.XXXXX`
 find -L $test  -type f  -exec grep -Iq . {} \; -print > $tmp_file
 #echo "/usr/local/${test}/results_hammerdb_*.csv" >> $tmp_file
-egrep  "out|csv|report" $tmp_file | egrep -v "Hammerdb" | grep -v tcl | tar cf /tmp/results_hammerdb_${test}_${to_tuned_setting}.tar --files-from=/dev/stdin
+grep  -E "out|csv|report" $tmp_file | grep -v "Hammerdb" | grep -v tcl | tar cf /tmp/results_hammerdb_${test}_${to_tuned_setting}.tar --files-from=/dev/stdin
 csv_file=`grep  "csv" $tmp_file | grep -v "Hammerdb"`
 ${TOOLS_BIN}/csv_to_json $to_json_flags --csv_file $csv_file --output_file results_hammer.json
 $TOOLS_BIN/verify_results $to_verify_flags --schema_file $script_dir/results_schema.py --class_name Hammerdb_Results --file results_hammer.json

hammerdb/hammerdb.json

@@ -6,6 +6,7 @@
             "bc",
             "git",
             "unzip",
+            "wget",
             "zip"
         ],
         "pip": [

hammerdb/hammerdb_mssql.json

@@ -0,0 +1,15 @@
+{
+    "dependencies": {
+        "rhel": [
+            "mssql-server",
+            "unixODBC-devel",
+            "gdb",
+            "cyrus-sasl",
+            "libatomic",
+            "lsof"
+        ],
+        "pip": [
+            "typing-extensions"
+        ]
+    }
+}

hammerdb/hammerdb_scripts/mariadb/build_mariadb.tcl

@@ -0,0 +1,25 @@
+#!/bin/tclsh
+#
+
+puts "SETTING CONFIGURATION"
+
+global complete
+proc wait_to_complete {} {
+global complete
+set  complete [vucomplete]
+if {!$complete} {after 5000  wait_to_complete} else { exit } 
+}
+
+
+dbset db mysql
+diset connection mysql_host 127.0.0.1
+diset connection mysql_port 3306
+diset tpcc mysql_count_ware 500
+diset tpcc mysql_partition true
+diset tpcc mysql_num_vu 50
+diset tpcc mysql_pass mysql
+diset tpcc mysql_storage_engine innodb
+print dict
+buildschema
+wait_to_complete
+vwait forever

hammerdb/hammerdb_scripts/mariadb/build_mariadb_tpcc.sh

@@ -0,0 +1,88 @@
+#!/bin/sh
+
+export PATH=$PATH:.:
+## Stop and start the mariadb server and drop and create the database
+LD_LIBRARY_PATH=/lib64/mysql:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH
+
+## Setting up variables for the run and Collecting system information ####
+Usercount="50"
+whc="500"
+Testname="HDB_tpcc_mariadb"
+benchmark_name="Hammerdb-tpcc"
+echo ${benchmark_name} > ${benchmark_run_dir}/user-benchmark-name.txt
+hostnm=`hostname -f`
+hostip=`hostname -i`
+numcpu=`nproc`
+totmem=`cat /proc/meminfo |grep "MemTotal:" | awk '{print $2}'`
+krel=`uname -r`      ## Kernel
+
+usage()
+{
+  echo "Usage:
+        ./build_mariadb_tpcc.sh [-h] [-u users] [-w warehouse count] 
+
+        Usage:
+        -h help
+        -u # of users (Default - 50)
+        -w warehouse count (Default - 500)
+
+       Examples:
+        ./build_mariadb_tpcc.sh -u "10" -w 50
+            Build 50 warehouse tpcc database with 10 users
+  "
+}
+
+while [ $# -gt 0 ]
+do
+case $1 in
+        -h) usage;
+            exit;
+            ;;
+        -w) whc=$2
+            shift 2
+            ;;
+        -u) Usercount=$2
+            shift 2
+            ;;
+         *) usage;
+            exit;
+            ;;
+esac
+done
+
+
+
+## Updating the value of buffer_pool_size based on available memory
+## Setting bufferpool to half of physical memory if memory is less than 128G else set to 64G
+totmem_BP=`expr $totmem / 1024`
+totmem_BP=`expr $totmem_BP / 2`
+
+if [ $totmem_BP -lt 64000 ]
+then
+   sed -i "s/^innodb_buffer_pool_size=.*/innodb_buffer_pool_size=${totmem_BP}M/" my.cnf
+else
+   sed -i "s/^innodb_buffer_pool_size=.*/innodb_buffer_pool_size=64000M/" my.cnf
+fi
+
+/usr/bin/cp -f my.cnf /etc/my.cnf
+
+### Shutting down and starting Mariadb instance ####
+systemctl restart mariadb.service
+echo "Restarted DB"
+sleep 60
+
+# Define number of warehouses and user count for the build
+
+
+echo "Dropping database tpcc"
+mysql -pabcdefgh -e 'drop database tpcc;'
+
+echo "Building ${whc} warehouses with ${Usercount} users"
+sed -i "s/^diset connection mysql_host.*/diset connection mysql_host ${hostip}/" build_mariadb.tcl
+sed -i "s/^diset tpcc mysql_count_ware.*/diset tpcc mysql_count_ware ${whc}/" build_mariadb.tcl
+sed -i "s/^diset tpcc mysql_num_vu.*/diset tpcc mysql_num_vu ${Usercount}/" build_mariadb.tcl
+
+./hammerdbcli auto build_mariadb.tcl > build_mariadb_${Testname}_${whc}WH_${Usercount}.out
+
+echo "Mariadb TPCC database with ${whc} build done"

hammerdb/hammerdb_scripts/mariadb/my.cnf

@@ -0,0 +1,69 @@
+[mysqld]
+#large-pages
+skip-log-bin
+datadir=/perf1/mysql/data
+port=3306
+skip-networking=0
+skip-bind-address
+skip-grant-tables
+
+# general
+max_connections=4000
+table_open_cache=8000
+#table_open_cache_instances=16
+back_log=1500
+#default_password_lifetime=0
+ssl=0
+performance_schema=OFF
+max_prepared_stmt_count=128000
+skip_log_bin=1
+character_set_server=latin1
+collation_server=latin1_swedish_ci
+transaction_isolation=REPEATABLE-READ
+
+# files
+innodb_file_per_table
+innodb_log_file_size=1024M
+innodb_log_files_in_group=32
+innodb_open_files=4000
+
+# buffers
+innodb_buffer_pool_size=16384M
+innodb_buffer_pool_instances=16
+innodb_log_buffer_size=64M
+
+# tune
+innodb_doublewrite=0
+innodb_thread_concurrency=0
+innodb_flush_log_at_trx_commit=0
+innodb_max_dirty_pages_pct=90
+#innodb_max_dirty_pages_pct_lwm=10
+
+join_buffer_size=32K
+sort_buffer_size=32K
+innodb_use_native_aio=1
+#innodb_stats_persistent=1
+innodb_spin_wait_delay=6
+
+#innodb_max_purge_lag_delay=300000
+#innodb_max_purge_lag=0
+innodb_flush_method=O_DIRECT
+#innodb_checksum_algorithm=none
+#innodb_io_capacity=4000
+#innodb_io_capacity_max=20000
+#innodb_lru_scan_depth=9000
+innodb_change_buffering=none
+#innodb_read_only=0
+#innodb_page_cleaners=4
+#innodb_undo_log_truncate=off
+
+# perf special
+innodb_adaptive_flushing=1
+#innodb_flush_neighbors=0
+innodb_read_io_threads=16
+innodb_write_io_threads=16
+innodb_purge_threads=4
+innodb_adaptive_hash_index=0
+
+# monitoring
+#innodb_monitor_enable='%'

hammerdb/hammerdb_scripts/mariadb/run_mariadb_tpcc.sh

@@ -0,0 +1,111 @@
+#!/bin/sh
+export PATH=$PATH:.:
+
+LD_LIBRARY_PATH=/lib64/mariadb:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH
+
+## Set variables for the run
+storagetype="Storage not specified"
+Usercount="10 20 40 80 100"
+whc="500"
+DBRESTARTUP="n"
+Testname="HDB_tpcc_mariadb"
+
+usage()
+{
+  echo "Usage:
+        run_mssql_tpcc.sh [-h] [-u users] [-t runlength] [-w warehouse count] [-s storage type]
+
+        Usage:
+        -h help
+        -u # of users (Default - "10 20 40 80 100")
+        -t runlength in minutes (Default - 15 minuteS)
+        -w warehouse count (Default - 500 )
+        -s storage type ( Default - "Storage not speficied" )
+
+       Examples:
+        run_mariadb_tpcc.sh -u "10 20 30"
+            Do runs with 10, 20 and 30 users
+        run_mariadb_tpcc.sh -s "nvme"
+            Do a 10 user, 500 warehouse, 15 minute run with storage type as "nvme"
+        run_mariadb_tpcc.sh -w 1000 -u "10 20 40 80 100" -s "iscsi"
+            Do a 1000 warehouse run with 10 20 40 80 and 100 users and storage type specified as iscsi
+  "
+}
+
+while [ $# -gt 0 ]
+do
+case $1 in
+        -h) usage;
+            exit;
+            ;;
+        -w) whc=$2
+            shift 2
+            ;;
+        -u) Usercount=$2
+            shift 2
+            ;;
+        -s) storagetype=$2
+            shift 2
+            ;;
+        -t) RUNLENGTH=$2
+            shift 2
+            ;;
+        *) usage;
+            exit;
+            ;;
+esac
+done
+
+mkdir -p results
+
+if [ -z "${benchmark_results_dir}" ]
+then
+benchmark_results_dir=`pwd`"/results"
+fi
+
+if [ -z "${benchmark_run_dir}" ]
+then
+benchmark_run_dir=`pwd`"/results"
+fi
+
+#echo ${storagetype}
+#echo ${Usercount}
+#echo ${benchmark_results_dir}
+#echo ${whc}
+
+
+## Collecting system information
+benchmark_name="Hammerdb-tpcc"
+benchmark_ver="HammerDB-3.2"
+hostnm=`hostname -f`
+hostip=`hostname -i`
+numcpu=`nproc`
+totmem=`cat /proc/meminfo |grep "MemTotal:" | awk '{print $2}'`
+krel=`uname -r`      ## Kernel
+
+
+# Set Host IP and Warehouse Count in the tcl file
+sed -i "s/^diset tpcc mysql_count_ware.*/diset tpcc mysql_count_ware ${whc}/" runtest_mariadb.tcl
+
+echo "StartTime,EndTime,Hostname,Kernel,Database,DBVer,Cpus,Memory,StorageType,Users,Tpm" > user-benchmark-result.csv
+mariaver=`mysql -V |awk '{print $5}' | sed -e 's/,//'`
+
+
+for uc in ${Usercount}
+do
+    echo "Restarting Database before each run"
+    systemctl restart mariadb.service
+    starttime=`date +%Y.%m.%d.%T`
+    echo "Run with  ${uc} users"
+    sed -i "s/^vuset.*/vuset vu ${uc}/" runtest_mariadb.tcl
+    ./hammerdbcli auto runtest_mariadb.tcl > test_mariadb_${Testname}_${uc}.out 2>&1
+    endtime=`date +%Y.%m.%d.%T`
+    grep RESULT test_mariadb_${Testname}_${uc}.out
+    tpm=`grep TPM test_mariadb_${Testname}_${uc}.out | awk '{print $7}'`
+    echo "\"${starttime}\",\"${endtime}\",\"${hostnm}\",\"${krel}\",\"Mariadb\",\"${mariaver}\",\"${numcpu}\",\"${totmem}\",\"${storagetype}\",\"${uc}\",\"${tpm}\"" >> user-benchmark-result.csv
+    cp user-benchmark-result.csv $benchmark_results_dir
+    cp /etc/my.cnf $benchmark_results_dir
+    cp test_mariadb_${Testname}_${uc}.out $benchmark_results_dir
+done
+systemctl stop mariadb.service