Install update

hammerdb/hammerdb

@@ -23,6 +23,10 @@ log_mount=""
 pcp=""
 pdir=""
 script_dir=$(realpath $(dirname $0))
+export script_dir
+hmd_dir=${script_dir}/hammerdb-tpcc/Hammerdb
+mkdir -p $hmd_dir
+export hmd_dir
 
 curdir=`pwd`
 if [[ $0 == "./"* ]]; then
@@ -129,7 +133,8 @@ usage()
 
 ${TOOLS_BIN}/gather_data ${curdir}
 source ${TOOLS_BIN}/general_setup "$@"
-export to_no_pkg_install
+export to_pkg_tool_flags
+
 
 unset DISPLAY
 users_to_run=""
@@ -289,7 +294,7 @@ if [[ $to_tuned_setting != "none" ]]; then
 	tuned_original=`tuned-adm active | cut -d' ' -f4`
 	tuned-adm profile $to_tuned_setting
 fi
-pushd /usr/local > /dev/null
+pushd $hmd_dir > /dev/null
 rm -rf $test
 popd > /dev/null
 cpdir=""
@@ -303,25 +308,16 @@ echo ./run_hammerdb -m /perf1 -t ${test} ${users_to_run} ${warehouses} ${log_mou
 chmod 755 run_this
 ./run_this
 
-cd /usr/local
-mv HammerDB $test
-
-${TOOLS_BIN}/move_data $curdir $test
-pushd $test
-files=`ls test*out`
-if [ $? -ne 0 ]; then
-	echo Failed >> test_results_report
-else
-	echo Ran >> test_results_report
-fi
-
+#pwd
+#echo $hmd_dir $test
+rm -rf $test
+mv $hmd_dir $test
 cp /tmp/hammerdb.out .
-popd
 
 tmp_file=`mktemp /tmp/hammer_data.XXXXX`
 find -L $test  -type f  -exec grep -Iq . {} \; -print > $tmp_file
 #echo "/usr/local/${test}/results_hammerdb_*.csv" >> $tmp_file
-egrep  "out|csv|report" $tmp_file | egrep -v "Hammerdb" | grep -v tcl | tar cf /tmp/results_hammerdb_${test}_${to_tuned_setting}.tar --files-from=/dev/stdin
+grep  -E "out|csv|report" $tmp_file | grep -v "Hammerdb" | grep -v tcl | tar cf /tmp/results_hammerdb_${test}_${to_tuned_setting}.tar --files-from=/dev/stdin
 csv_file=`grep  "csv" $tmp_file | grep -v "Hammerdb"`
 ${TOOLS_BIN}/csv_to_json $to_json_flags --csv_file $csv_file --output_file results_hammer.json
 $TOOLS_BIN/verify_results $to_verify_flags --schema_file $script_dir/results_schema.py --class_name Hammerdb_Results --file results_hammer.json

hammerdb/hammerdb.json

@@ -6,6 +6,7 @@
             "bc",
             "git",
             "unzip",
+            "wget",
             "zip"
         ],
         "pip": [

hammerdb/hammerdb_mssql.json

@@ -0,0 +1,15 @@
+{
+    "dependencies": {
+        "rhel": [
+            "mssql-server",
+            "unixODBC-devel",
+            "gdb",
+            "cyrus-sasl",
+            "libatomic",
+            "lsof"
+        ],
+        "pip": [
+            "typing-extensions"
+        ]
+    }
+}

hammerdb/hammerdb_scripts/mariadb/Readme_mariadb_rhel7

@@ -0,0 +1,153 @@
+1) Make sure you have MariaDB already installed. Use the version that ships with RHEL  
+
+In RHEL8 you will need the rhel8-Buildroot repo
+
+In RHEL7 you will need the 
+
+yum -y install mariadb.x86_64 mariadb-common.x86_64 mariadb-config.x86_64 \
+mariadb-devel.x86_64 mariadb-errmsg.x86_64 mariadb-libs.x86_64 mariadb-server.x86_64 mariadb-server-utils.x86_64 
+
+2) Install the dev tools and the dependencies 
+
+yum -y install gcc gcc-c++ autoconf automake make libtool \
+zlib zlib-devel openssl-devel nfs-utils
+
+mkdir /shak
+mount perf1:/shak /shak
+cd /shak/HammerDB/
+./HammerDB-3.2-Linux-x86-64-Install 
+
+
+cd /usr/local/HammerDB-3.0/
+cp /shak/HammerDB/tpcc/mariadb/* .
+cp /shak/HammerDB/tpcc/mariadb/my.cnf /etc/my.cnf
+
+Create directories that are defined in the file 
+datadir=/perf1/mysql/data
+
+In this case, it is /perf1/mysql/data
+If you want to use a storage device, create the file system.
+
+mkfs -t ext4 /dev/vda
+mkdir /perf1
+mount /dev/vda /perf1
+
+#Create directory and change permissions
+
+mkdir -p /perf1/mysql/data
+chmod 777 /perf1/mysql/data
+
+With this version of MariaDB, you need to setup a password for root user
+and use the password during the runs
+
+# Start instance. This will create base tables
+### IMPORTANT - This will take some time because the updated my.cnf file creates a bunch of log files so wait for 
+### a couple of minutes
+
+mysqld_safe --user=root --basedir=/usr --skip-grant-tables &
+
+### Wait for 2 minutes atleast depending on the time of the storage or just tail the err file to see if the db is up
+
+tail -f /perf1/mysql/data/`hostname`.err 
+
+Look for this line - 181130 11:50:04 [Note] /usr/libexec/mysqld: ready for connections.
+
+
+# Shutdown database
+mysqladmin -f -uroot -p100yard- shutdown
+
+
+# Configure stored procedures ### Important step
+mysql_install_db --defaults-file=/etc/my.cnf
+
+# Change ownership of the directory to mysql user
+chown -R mysql:mysql /perf1/mysql
+
+# Restart the database using this command This will run it as mysql user
+
+mysqld_safe --user=root --basedir=/usr --skip-grant-tables &
+
+# This command will set up password for root user
+
+/usr/bin/mysqladmin -u root password '100yard-'
+
+Copy the required mysql library file libmysqlclient.so.20 to /lib64/mysql directory
+
+cp /shak/HammerDB/tpcc/mariadb/libmysqlclient.so.20 /lib64/mysql/
+
+If the /lib64/mysql directory is missing, create it and copy the file
+
+# Make sure these lines are in the build_and_run_mariadb.sh script
+
+LD_LIBRARY_PATH=/lib64/mysql:$LD_LIBRARY_PATH:
+export LD_LIBRARY_PATH
+
+### Install Hammerdb kit
+cd /shak/HammerDB/
+
+### Check and install the latest Hammerdb version. There maybe a newer version than the one in the command below
+./HammerDB-3.2-Linux-x86-64-Install
+
+
+
+# Copy the mariadb kit and run files
+cd /usr/local/HammerDB-3.2/
+
+cp /shak/HammerDB/tpcc/mariadb/* .
+
+## A new module has helped improve the performance of the tests and it's been added to these new files. But if you decide to
+## test with the new code, you have to collect new baselines. Do the following steps to use the new tcl files for testing
+
+cp new_build_mariadb.tcl build_mariadb.tcl
+cp new_runtest_mariadb.tcl runtest_mariadb.tcl
+
+
+## IMPORTANT - This workload needs the IP address of the host where the database is running. Even
+# for local host, it won't take 127.0.0.1. So please update both the .tcl files 
+
+-  build_mariadb.tcl
+-  runtest_mariadb.tcl
+
+# Edit this file for user and warehouse count and run it. It will build the schema and run the workload
+
+./build_and_run_mariadb.sh
+
+
+
+Below are some of the setting and commands to do the build
+
+##  Create database with the following command (drop if it's a re-run)
+mysqladmin -f -uroot -p100yard- drop tpcc
+mysqladmin -uroot -p100yard- create tpcc
+
+
+./hammerdbcli
+dbset db mysql
+diset connection mysql_host 10.16.29.121
+diset connection mysql_port 3306
+diset tpcc mysql_count_ware 800
+diset tpcc mysql_partition true
+diset tpcc mysql_num_vu 64
+diset tpcc mysql_storage_engine innodb
+print dict
+buildschema
+
+
+dbset db mysql
+diset connection mysql_host 10.16.29.121
+diset connection mysql_port 3306
+diset tpcc mysql_driver timed
+diset tpcc mysql_rampup 2
+diset tpcc mysql_duration 5
+vuset logtotemp 1
+loadscript
+vuset vu 10
+vucreate
+vurun
+
+
+DB size
+100WH - 8G
+200WH - 17G
+500WH - 50G
+800WH - 70G

hammerdb/hammerdb_scripts/mariadb/Readme_mariadb_rhel8

@@ -0,0 +1,126 @@
+1) Make sure you have MariaDB already installed. Use the version that ships with RHEL  
+
+In RHEL8 you will need the rhel8-Buildroot repo
+
+dnf -y install mariadb.x86_64 mariadb-common.x86_64 mariadb-devel.x86_64 \
+mariadb-errmsg.x86_64 mariadb-server.x86_64 mariadb-server-utils.x86_64 mysql-libs.x86_64
+
+mkdir /shak
+mount perf1:/shak /shak
+
+cd /shak/HammerDB
+./HammerDB-3.2-Linux-x86-64-Install 
+
+
+cd /usr/local/HammerDB-3.2/
+cp /shak/HammerDB/tpcc/mariadb/* .
+cp /shak/HammerDB/tpcc/mariadb/my.cnf /etc/my.cnf
+
+Create directories that are defined in the file 
+datadir=/perf1/mysql/data
+
+In this case, it is /perf1/mysql/data
+If you want to use a storage device, create the file system.
+
+mkfs -t ext4 /dev/vda
+mkdir /perf1
+mount /dev/vda /perf1
+
+#Create directory and change permissions
+
+mkdir -p /perf1/mysql/data
+chown -R mysql:mysql /perf1/mysql
+
+systemctl start mariadb.service
+
+## Hammerdb uses password to connect to the database so run the following command to setup password for root
+
+echo "UPDATE mysql.user SET Password=PASSWORD('100yard-')  WHERE User='root';" > input
+echo "flush privileges;" >> input
+echo "exit" >> input
+
+mysql -u root < input
+
+
+Copy the required mysql library file libmysqlclient.so.20 to /lib64/mariadb directory
+
+cp /shak/HammerDB/tpcc/mariadb/libmysqlclient.so.20 /lib64/mariadb/
+
+If the /lib64/mariadb directory is missing, create it and copy the file
+
+# Make sure these lines are in the build_and_run_mariadb.sh script
+
+LD_LIBRARY_PATH=/lib64/mysql:$LD_LIBRARY_PATH:
+export LD_LIBRARY_PATH
+
+
+# Edit this file for user and warehouse count and run it. It will build the schema and run the workload
+
+./build_mariadb_tpcc.sh
+
+Run the workload with run_mariadb_tpcc.sh. To see options run
+
+./run_mariadb_tpch.sh -h
+
+
+e.g
+./run_mariadb_tpcc.sh -s "nvme" -u "10 20 40 80 100"
+
+./run_mariadb_tpcc.sh -s "iscsi" -u "10"
+
+
+----------------------------------------------------------------------------------------------------------------------------
+Running Workload with pbench
+
+If you want to run the workload with pbench harness with a iteration list
+
+pbench-user-benchmark -C "Maridb_tuned_TP_HTon_40P_256Gmem_with_csv" --iteration-list=./run_params -- ./run_mariadb_tpcc.sh
+
+where run_params has the following entries
+10U NVME 10
+20U NVME 20
+
+If you just want to do one run without the iteration file
+
+pbench-user-benchmark -C "Maridb_tuned_TP_HTon_40P_256Gmem_with_csv"  -- ./run_mariadb_tpcc.sh nvme "10"
+
+
+
+----------------------------------------------------------------- End of Document --------------------
+Below are some of the setting and commands to do the build
+
+##  Create database with the following command (drop if it's a re-run)
+mysqladmin -f -uroot -p100yard- drop tpcc
+mysqladmin -uroot -p100yard- create tpcc
+
+
+./hammerdbcli
+dbset db mysql
+diset connection mysql_host 10.16.29.121
+diset connection mysql_port 3306
+diset tpcc mysql_count_ware 800
+diset tpcc mysql_partition true
+diset tpcc mysql_num_vu 64
+diset tpcc mysql_storage_engine innodb
+print dict
+buildschema
+
+
+dbset db mysql
+diset connection mysql_host 10.16.29.121
+diset connection mysql_port 3306
+diset tpcc mysql_driver timed
+diset tpcc mysql_rampup 2
+diset tpcc mysql_duration 5
+vuset logtotemp 1
+loadscript
+vuset vu 10
+vucreate
+vurun
+
+
+DB size
+100WH - 8G
+200WH - 17G
+500WH - 50G
+800WH - 70G

hammerdb/hammerdb_scripts/mariadb/build_mariadb.tcl

@@ -0,0 +1,25 @@
+#!/bin/tclsh
+#
+
+puts "SETTING CONFIGURATION"
+
+global complete
+proc wait_to_complete {} {
+global complete
+set  complete [vucomplete]
+if {!$complete} {after 5000  wait_to_complete} else { exit } 
+}
+
+
+dbset db mysql
+diset connection mysql_host 127.0.0.1
+diset connection mysql_port 3306
+diset tpcc mysql_count_ware 500
+diset tpcc mysql_partition true
+diset tpcc mysql_num_vu 50
+diset tpcc mysql_pass mysql
+diset tpcc mysql_storage_engine innodb
+print dict
+buildschema
+wait_to_complete
+vwait forever