From 7a125381683820acdc57792256425632cb56cb0a Mon Sep 17 00:00:00 2001
From: vrofze <vrofze@gmail.com>
Date: Mon, 5 Mar 2018 16:37:54 +0800
Subject: [PATCH 01/27] Add secondary_ranges in src/lua/oltp_common.lua

---
 src/lua/oltp_common.lua     | 13 +++++++++++++
 src/lua/oltp_read_only.lua  |  2 ++
 src/lua/oltp_read_write.lua |  2 ++
 tests/t/script_oltp_help.t  |  1 +
 4 files changed, 18 insertions(+)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index b0a77e377..d8fa3a34d 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -41,6 +41,8 @@ sysbench.cmdline.options = {
       {"Number of point SELECT queries per transaction", 10},
    simple_ranges =
       {"Number of simple range SELECT queries per transaction", 1},
+   secondary_ranges =
+      {"Number of secondary range SELECT queries per transaction", 1},
    sum_ranges =
       {"Number of SELECT SUM() queries per transaction", 1},
    order_ranges =
@@ -251,6 +253,9 @@ local stmt_defs = {
    simple_ranges = {
       "SELECT c FROM sbtest%u WHERE id BETWEEN ? AND ?",
       t.INT, t.INT},
+   secondary_ranges = {
+      "SELECT c FROM sbtest%u WHERE k BETWEEN ? AND ?",
+      t.INT, t.INT},
    sum_ranges = {
       "SELECT SUM(k) FROM sbtest%u WHERE id BETWEEN ? AND ?",
        t.INT, t.INT},
@@ -322,6 +327,10 @@ function prepare_simple_ranges()
    prepare_for_each_table("simple_ranges")
 end
 
+function prepare_secondary_ranges()
+   prepare_for_each_table("secondary_ranges")
+end
+
 function prepare_sum_ranges()
    prepare_for_each_table("sum_ranges")
 end
@@ -440,6 +449,10 @@ function execute_simple_ranges()
    execute_range("simple_ranges")
 end
 
+function execute_secondary_ranges()
+   execute_range("secondary_ranges")
+end
+
 function execute_sum_ranges()
    execute_range("sum_ranges")
 end
diff --git a/src/lua/oltp_read_only.lua b/src/lua/oltp_read_only.lua
index 86c0fc8bf..9f57f142c 100755
--- a/src/lua/oltp_read_only.lua
+++ b/src/lua/oltp_read_only.lua
@@ -31,6 +31,7 @@ function prepare_statements()
 
    if sysbench.opt.range_selects then
       prepare_simple_ranges()
+      prepare_secondary_ranges()
       prepare_sum_ranges()
       prepare_order_ranges()
       prepare_distinct_ranges()
@@ -46,6 +47,7 @@ function event()
 
    if sysbench.opt.range_selects then
       execute_simple_ranges()
+      execute_secondary_ranges()
       execute_sum_ranges()
       execute_order_ranges()
       execute_distinct_ranges()
diff --git a/src/lua/oltp_read_write.lua b/src/lua/oltp_read_write.lua
index f5f05ce50..4a164ab59 100755
--- a/src/lua/oltp_read_write.lua
+++ b/src/lua/oltp_read_write.lua
@@ -31,6 +31,7 @@ function prepare_statements()
 
    if sysbench.opt.range_selects then
       prepare_simple_ranges()
+      prepare_secondary_ranges()
       prepare_sum_ranges()
       prepare_order_ranges()
       prepare_distinct_ranges()
@@ -50,6 +51,7 @@ function event()
 
    if sysbench.opt.range_selects then
       execute_simple_ranges()
+      execute_secondary_ranges()
       execute_sum_ranges()
       execute_order_ranges()
       execute_distinct_ranges()
diff --git a/tests/t/script_oltp_help.t b/tests/t/script_oltp_help.t
index 6052fb313..19a2872f4 100644
--- a/tests/t/script_oltp_help.t
+++ b/tests/t/script_oltp_help.t
@@ -21,6 +21,7 @@ OLTP usage information test
     --table_size=N                Number of rows per table [10000]
     --pgsql_variant=STRING        Use this PostgreSQL variant when running with the PostgreSQL driver. The only currently supported variant is 'redshift'. When enabled, create_secondary is automatically disabled, and delete_inserts is set to 0
     --simple_ranges=N             Number of simple range SELECT queries per transaction [1]
+    --secondary_ranges=N          Number of secondary range SELECT queries per transaction [1]
     --order_ranges=N              Number of SELECT ORDER BY queries per transaction [1]
     --range_selects[=on|off]      Enable/disable all range SELECT queries [on]
     --point_selects=N             Number of point SELECT queries per transaction [10]

From ca1e70f62023d11e5e8fb7dffe28c4d501c2790d Mon Sep 17 00:00:00 2001
From: vrofze <vrofze@gmail.com>
Date: Mon, 5 Mar 2018 17:35:30 +0800
Subject: [PATCH 02/27] Use limit instead of between in secondary_ranges

---
 src/lua/oltp_common.lua | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index d8fa3a34d..afedf3b64 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -254,7 +254,7 @@ local stmt_defs = {
       "SELECT c FROM sbtest%u WHERE id BETWEEN ? AND ?",
       t.INT, t.INT},
    secondary_ranges = {
-      "SELECT c FROM sbtest%u WHERE k BETWEEN ? AND ?",
+      "SELECT c FROM sbtest%u WHERE k >= ? AND ? >=0 LIMIT %u",
       t.INT, t.INT},
    sum_ranges = {
       "SELECT SUM(k) FROM sbtest%u WHERE id BETWEEN ? AND ?",
@@ -289,7 +289,11 @@ end
 
 function prepare_for_each_table(key)
    for t = 1, sysbench.opt.tables do
-      stmt[t][key] = con:prepare(string.format(stmt_defs[key][1], t))
+      if key == "secondary_ranges" then
+         stmt[t][key] = con:prepare(string.format(stmt_defs[key][1], t, sysbench.opt.range_size))
+      else
+         stmt[t][key] = con:prepare(string.format(stmt_defs[key][1], t))
+      end
 
       local nparam = #stmt_defs[key] - 1
 

From 47aaab40ae87757f8ba7413fa11ec75cec704d63 Mon Sep 17 00:00:00 2001
From: vrofze <vrofze@gmail.com>
Date: Fri, 23 Mar 2018 16:18:53 +0800
Subject: [PATCH 03/27] use text file as source data

---
 src/db_driver.c                    |  2 +-
 src/lua/internal/sysbench.rand.lua |  8 +++++
 src/lua/oltp_common.lua            | 56 ++++++++++++++++++++++--------
 src/sb_rand.c                      | 54 +++++++++++++++++++++++++++-
 src/sb_rand.h                      |  4 +++
 src/sysbench.c                     | 16 ++++++++-
 src/sysbench.h                     |  1 +
 7 files changed, 124 insertions(+), 17 deletions(-)

diff --git a/src/db_driver.c b/src/db_driver.c
index 509d17344..b59b22b43 100644
--- a/src/db_driver.c
+++ b/src/db_driver.c
@@ -43,7 +43,7 @@
 #include "sb_ck_pr.h"
 
 /* Query length limit for bulk insert queries */
-#define BULK_PACKET_SIZE (512*1024)
+#define BULK_PACKET_SIZE (5120*1024)
 
 /* How many rows to insert before COMMITs (used in bulk insert) */
 #define ROWS_BEFORE_COMMIT 1000
diff --git a/src/lua/internal/sysbench.rand.lua b/src/lua/internal/sysbench.rand.lua
index 2ff6766a5..39c63972c 100644
--- a/src/lua/internal/sysbench.rand.lua
+++ b/src/lua/internal/sysbench.rand.lua
@@ -34,6 +34,7 @@ uint32_t sb_rand_unique(void);
 void sb_rand_str(const char *, char *);
 void sb_rand_varstr(char *, uint32_t, uint32_t);
 double sb_rand_uniform_double(void);
+void sb_file_str(char *, char *);
 ]]
 
 function sysbench.rand.uniform_uint64()
@@ -87,3 +88,10 @@ end
 function sysbench.rand.uniform_double()
    return ffi.C.sb_rand_uniform_double()
 end
+
+function sysbench.rand.filestr()
+   local buf1 = ffi.new("char[?]", 1024)
+   local buf2 = ffi.new("char[?]", 204800)
+   ffi.C.sb_file_str(buf1, buf2)
+   return ffi.string(buf1), ffi.string(buf2)
+end
\ No newline at end of file
diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index afedf3b64..254125f51 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -68,6 +68,8 @@ sysbench.cmdline.options = {
       {"Use a secondary index in place of the PRIMARY KEY", false},
    create_secondary =
       {"Create a secondary index in addition to the PRIMARY KEY", true},
+   use_file =
+      {"If use a text file as dataset", false},
    mysql_storage_engine =
       {"Storage engine, if MySQL is used", "innodb"},
    pgsql_variant =
@@ -151,6 +153,10 @@ function get_pad_value()
    return sysbench.rand.string(pad_value_template)
 end
 
+function get_str_value()
+   return sysbench.rand.filestr()
+end
+
 function create_table(drv, con, table_num)
    local id_index_def, id_def
    local engine_def = ""
@@ -188,15 +194,27 @@ function create_table(drv, con, table_num)
 
    print(string.format("Creating table 'sbtest%d'...", table_num))
 
-   query = string.format([[
-CREATE TABLE sbtest%d(
-  id %s,
-  k INTEGER DEFAULT '0' NOT NULL,
-  c CHAR(120) DEFAULT '' NOT NULL,
-  pad CHAR(60) DEFAULT '' NOT NULL,
-  %s (id)
-) %s %s]],
-      table_num, id_def, id_index_def, engine_def, extra_table_options)
+   if sysbench.opt.use_file then
+      query = string.format([[
+   CREATE TABLE sbtest%d(
+     id %s,
+     k INTEGER DEFAULT '0' NOT NULL,
+     c VARCHAR(512) DEFAULT '' NOT NULL,
+     pad VARCHAR(1024) DEFAULT '' NOT NULL,
+     %s (id)
+   ) %s %s]],
+         table_num, id_def, id_index_def, engine_def, extra_table_options)
+   else
+      query = string.format([[
+   CREATE TABLE sbtest%d(
+     id %s,
+     k INTEGER DEFAULT '0' NOT NULL,
+     c CHAR(120) DEFAULT '' NOT NULL,
+     pad CHAR(60) DEFAULT '' NOT NULL,
+     %s (id)
+   ) %s %s]],
+         table_num, id_def, id_index_def, engine_def, extra_table_options)
+   end
 
    con:query(query)
 
@@ -218,15 +236,19 @@ CREATE TABLE sbtest%d(
 
    for i = 1, sysbench.opt.table_size do
 
-      c_val = get_c_value()
-      pad_val = get_pad_value()
+      if sysbench.opt.use_file then
+         c_val, pad_val = get_str_value()
+      else
+         c_val = get_c_value()
+         pad_val = get_pad_value()
+      end
 
       if (sysbench.opt.auto_inc) then
-         query = string.format("(%d, '%s', '%s')",
+         query = string.format("(%d, \"%s\", \"%s\")",
                                sysbench.rand.default(1, sysbench.opt.table_size),
                                c_val, pad_val)
       else
-         query = string.format("(%d, %d, '%s', '%s')",
+         query = string.format("(%d, %d, \"%s\", \"%s\")",
                                i,
                                sysbench.rand.default(1, sysbench.opt.table_size),
                                c_val, pad_val)
@@ -481,9 +503,15 @@ end
 
 function execute_non_index_updates()
    local tnum = get_table_num()
+   local c_val;
 
    for i = 1, sysbench.opt.non_index_updates do
-      param[tnum].non_index_updates[1]:set_rand_str(c_value_template)
+      if sysbench.opt.use_file then
+         c_val = get_str_value()
+      else
+         c_val = get_c_value()
+      end
+      param[tnum].non_index_updates[1]:set(c_val)
       param[tnum].non_index_updates[2]:set(get_id())
 
       stmt[tnum].non_index_updates:execute()
diff --git a/src/sb_rand.c b/src/sb_rand.c
index 712335ba6..1a7e3910b 100644
--- a/src/sb_rand.c
+++ b/src/sb_rand.c
@@ -40,6 +40,8 @@
 #endif
 
 #include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
 #ifdef HAVE_STRING_H
 # include <string.h>
 #endif
@@ -50,6 +52,7 @@
 # include <math.h>
 #endif
 
+#include "sysbench.h"
 #include "sb_options.h"
 #include "sb_rand.h"
 #include "sb_logger.h"
@@ -85,7 +88,6 @@ static sb_arg_t rand_args[] =
   SB_OPT("rand-zipfian-exp",
          "shape parameter (exponent, theta) for the Zipfian distribution",
          "0.8", DOUBLE),
-
   SB_OPT_END
 };
 
@@ -520,6 +522,56 @@ uint32_t sb_rand_zipfian(uint32_t a, uint32_t b)
     sb_rand_zipfian_int(b - a + 1, zipf_exp, zipf_s, zipf_hIntegralX1) - 1;
 }
 
+static FILE *file = NULL;
+
+int sb_file_init()
+{
+  assert(sb_globals.filename != NULL);
+
+  file = fopen(sb_globals.filename, "r");
+  if (file == NULL)
+  {
+    log_text(LOG_FATAL, "Invalid filename: %s", sb_globals.filename);
+    return 1;
+  }
+  return 0;
+}
+
+void sb_file_str(char* buf1, char* buf2)
+{
+  assert(file != NULL);
+
+  char* buf = NULL;
+  size_t len = 0;
+  ssize_t read = getline(&buf, &len, file);
+  if (read != -1)
+  {
+    int pos = 0;
+    while (pos < read) {
+      if (buf[pos++] == '\t') {
+        memcpy(buf1, buf, pos - 1);
+        memcpy(buf2, buf + pos, read - pos - 1);
+        break;
+      }
+    }
+
+    if (pos >= read) {
+      memcpy(buf2, buf, read);
+    }
+  }
+
+  free(buf);
+}
+
+void sb_file_done()
+{
+  if (file)
+  {
+    free(file);
+  }
+}
+
+
 /*
   H(x) is defined as
 
diff --git a/src/sb_rand.h b/src/sb_rand.h
index 32f76342a..a8f65cf7a 100644
--- a/src/sb_rand.h
+++ b/src/sb_rand.h
@@ -73,4 +73,8 @@ uint32_t sb_rand_unique(void);
 void sb_rand_str(const char *, char *);
 uint32_t sb_rand_varstr(char *, uint32_t, uint32_t);
 
+int sb_file_init(void);
+void sb_file_str(char *, char *);
+void sb_file_done(void);
+
 #endif /* SB_RAND_H */
diff --git a/src/sysbench.c b/src/sysbench.c
index d78a56a78..cbc1c265b 100644
--- a/src/sysbench.c
+++ b/src/sysbench.c
@@ -119,6 +119,7 @@ sb_arg_t general_args[] =
   SB_OPT("luajit-cmd", "perform LuaJIT control command. This option is "
          "equivalent to 'luajit -j'. See LuaJIT documentation for more "
          "information", NULL, STRING),
+  SB_OPT("filename", "whether to use file as data source", "", STRING),
 
   /* Deprecated aliases */
   SB_OPT("tx-rate", "deprecated alias for --rate", "0", INT),
@@ -1279,7 +1280,6 @@ static int checkpoint_cmp(const void *a_ptr, const void *b_ptr)
   return (int) (a - b);
 }
 
-
 static int init(void)
 {
   option_t *opt;
@@ -1392,6 +1392,15 @@ static int init(void)
     return 1;
   }
 
+  sb_globals.filename = sb_get_value_string("filename");
+  if (sb_globals.filename)
+  {
+    if (sb_file_init())
+    {
+      return 1;
+    }
+  }
+
   sb_globals.tx_rate = sb_get_value_int("tx-rate");
   if (sb_globals.tx_rate > 0)
   {
@@ -1616,6 +1625,11 @@ int main(int argc, char *argv[])
 
   sb_thread_done();
 
+  if (sb_globals.filename)
+  {
+    sb_file_done();
+  }
+
   free(timers);
   free(timers_copy);
 
diff --git a/src/sysbench.h b/src/sysbench.h
index 2fb2ad89d..ffd12e403 100644
--- a/src/sysbench.h
+++ b/src/sysbench.h
@@ -208,6 +208,7 @@ typedef struct
   int             warmup_time;  /* warmup time */
   uint64_t        nevents CK_CC_CACHELINE; /* event counter */
   const char      *luajit_cmd; /* LuaJIT command */
+  const char      *filename; /*if use a text file as dataset*/
 } sb_globals_t;
 
 extern sb_globals_t sb_globals CK_CC_CACHELINE;

From bdf0942ec7d5a38567b1c2ee68fe23c0fb16fbfd Mon Sep 17 00:00:00 2001
From: vrofze <vrofze@gmail.com>
Date: Tue, 27 Mar 2018 11:15:30 +0800
Subject: [PATCH 04/27] Use global variables in loop while read file

---
 src/lua/internal/sysbench.rand.lua |  5 +++--
 src/lua/oltp_common.lua            |  2 +-
 src/sb_rand.c                      | 31 +++++++++++++++++++-----------
 3 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/src/lua/internal/sysbench.rand.lua b/src/lua/internal/sysbench.rand.lua
index 39c63972c..e5472094f 100644
--- a/src/lua/internal/sysbench.rand.lua
+++ b/src/lua/internal/sysbench.rand.lua
@@ -89,9 +89,10 @@ function sysbench.rand.uniform_double()
    return ffi.C.sb_rand_uniform_double()
 end
 
+buf1 = ffi.new("char[?]", 1024)
+buf2 = ffi.new("char[?]", 4194304)
+
 function sysbench.rand.filestr()
-   local buf1 = ffi.new("char[?]", 1024)
-   local buf2 = ffi.new("char[?]", 204800)
    ffi.C.sb_file_str(buf1, buf2)
    return ffi.string(buf1), ffi.string(buf2)
 end
\ No newline at end of file
diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index 254125f51..8824515ad 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -200,7 +200,7 @@ function create_table(drv, con, table_num)
      id %s,
      k INTEGER DEFAULT '0' NOT NULL,
      c VARCHAR(512) DEFAULT '' NOT NULL,
-     pad VARCHAR(1024) DEFAULT '' NOT NULL,
+     pad MEDIUMTEXT DEFAULT '' NOT NULL,
      %s (id)
    ) %s %s]],
          table_num, id_def, id_index_def, engine_def, extra_table_options)
diff --git a/src/sb_rand.c b/src/sb_rand.c
index 1a7e3910b..4e534f61b 100644
--- a/src/sb_rand.c
+++ b/src/sb_rand.c
@@ -522,7 +522,9 @@ uint32_t sb_rand_zipfian(uint32_t a, uint32_t b)
     sb_rand_zipfian_int(b - a + 1, zipf_exp, zipf_s, zipf_hIntegralX1) - 1;
 }
 
-static FILE *file = NULL;
+static FILE* file = NULL;
+static char* str_buf;
+static size_t len;
 
 int sb_file_init()
 {
@@ -534,6 +536,9 @@ int sb_file_init()
     log_text(LOG_FATAL, "Invalid filename: %s", sb_globals.filename);
     return 1;
   }
+
+  len = 4194304 * sizeof(char);
+  str_buf = (char *)malloc(len);
   return 0;
 }
 
@@ -541,33 +546,37 @@ void sb_file_str(char* buf1, char* buf2)
 {
   assert(file != NULL);
 
-  char* buf = NULL;
-  size_t len = 0;
-  ssize_t read = getline(&buf, &len, file);
+  ssize_t read = getline(&str_buf, &len, file);
   if (read != -1)
   {
     int pos = 0;
     while (pos < read) {
-      if (buf[pos++] == '\t') {
-        memcpy(buf1, buf, pos - 1);
-        memcpy(buf2, buf + pos, read - pos - 1);
+      if (str_buf[pos] == '\t') {
+        memcpy(buf1, str_buf, pos);
+        buf1[pos] = 0;
+        memcpy(buf2, str_buf + pos + 1, read - pos - 2);
+        buf2[read - pos - 2] = 0;
         break;
       }
+      pos++;
     }
 
     if (pos >= read) {
-      memcpy(buf2, buf, read);
+      memcpy(buf2, str_buf, read);
     }
   }
-
-  free(buf);
 }
 
 void sb_file_done()
 {
   if (file)
   {
-    free(file);
+    fclose(file);
+  }
+
+  if (str_buf)
+  {
+    free(str_buf);
   }
 }
 

From 926c549d28f6d24dd097ec4aebf159f819a0b753 Mon Sep 17 00:00:00 2001
From: frost <vrofze@gmail.com>
Date: Mon, 13 Aug 2018 11:20:29 +0800
Subject: [PATCH 05/27] update oltp_common.lua

---
 src/lua/oltp_common.lua | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index 8824515ad..3423a238e 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -160,7 +160,7 @@ end
 function create_table(drv, con, table_num)
    local id_index_def, id_def
    local engine_def = ""
-   local extra_table_options = ""
+   local extra_table_options = "ROW_FORMAT=COMPRESSED"
    local query
 
    if sysbench.opt.secondary then
@@ -200,9 +200,9 @@ function create_table(drv, con, table_num)
      id %s,
      k INTEGER DEFAULT '0' NOT NULL,
      c VARCHAR(512) DEFAULT '' NOT NULL,
-     pad MEDIUMTEXT DEFAULT '' NOT NULL,
+     pad MEDIUMTEXT,
      %s (id)
-   ) %s %s]],
+   ) %s %s ROW_FORMAT=COMPRESSED]],
          table_num, id_def, id_index_def, engine_def, extra_table_options)
    else
       query = string.format([[

From 7cc3656682ccc1a095d9e8db83d53750a17d3f76 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Wed, 21 Sep 2022 11:48:15 +0800
Subject: [PATCH 06/27] remove ROW_FORMAT=COMPRESSED, and other minor fixes

---
 src/lua/oltp_common.lua |  7 ++++---
 src/sb_rand.c           |  1 +
 src/sysbench.c          | 12 ++----------
 3 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index 4e4e2ecf7..bb589fd0b 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -165,7 +165,6 @@ end
 function create_table(drv, con, table_num)
    local id_index_def, id_def
    local engine_def = ""
-   local extra_table_options = "ROW_FORMAT=COMPRESSED"
    local query
 
    if sysbench.opt.secondary then
@@ -206,7 +205,8 @@ function create_table(drv, con, table_num)
      pad MEDIUMTEXT,
      %s (id)
    ) %s %s ROW_FORMAT=COMPRESSED]],
-         table_num, id_def, id_index_def, engine_def, extra_table_options)
+         table_num, id_def, id_index_def, engine_def,
+         sysbench.opt.create_table_options)
    else
       query = string.format([[
    CREATE TABLE sbtest%d(
@@ -216,7 +216,8 @@ function create_table(drv, con, table_num)
      pad CHAR(60) DEFAULT '' NOT NULL,
      %s (id)
    ) %s %s]],
-         table_num, id_def, id_index_def, engine_def, extra_table_options)
+         table_num, id_def, id_index_def, engine_def,
+         sysbench.opt.create_table_options)
    end
 
    con:query(query)
diff --git a/src/sb_rand.c b/src/sb_rand.c
index 10eec042f..f67b59743 100644
--- a/src/sb_rand.c
+++ b/src/sb_rand.c
@@ -79,6 +79,7 @@ static sb_arg_t rand_args[] =
   SB_OPT("rand-zipfian-exp",
          "shape parameter (exponent, theta) for the Zipfian distribution",
          "0.8", DOUBLE),
+
   SB_OPT_END
 };
 
diff --git a/src/sysbench.c b/src/sysbench.c
index ba6298872..3ca5c64b8 100644
--- a/src/sysbench.c
+++ b/src/sysbench.c
@@ -119,7 +119,7 @@ sb_arg_t general_args[] =
   SB_OPT("luajit-cmd", "perform LuaJIT control command. This option is "
          "equivalent to 'luajit -j'. See LuaJIT documentation for more "
          "information", NULL, STRING),
-  SB_OPT("filename", "whether to use file as data source", "", STRING),
+  SB_OPT("filename", "use filename as data source", "", STRING),
 
   SB_OPT_END
 };
@@ -1290,6 +1290,7 @@ static int checkpoint_cmp(const void *a_ptr, const void *b_ptr)
   return (int) (a - b);
 }
 
+
 static int init(void)
 {
   option_t *opt;
@@ -1391,15 +1392,6 @@ static int init(void)
     }
   }
 
-  sb_globals.tx_rate = sb_get_value_int("tx-rate");
-  if (sb_globals.tx_rate > 0)
-  {
-    log_text(LOG_WARNING, "--tx-rate is deprecated, use --rate instead");
-    sb_opt_copy("rate", "tx-rate");
-  }
-  else
-    sb_globals.tx_rate = sb_get_value_int("rate");
-
   sb_globals.report_interval = sb_get_value_int("report-interval");
 
   sb_globals.n_checkpoints = 0;

From 3c4097df269deadce86ed9a172ee579853bcfa5e Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Wed, 21 Sep 2022 16:47:51 +0800
Subject: [PATCH 07/27] make select_random_ranges fetch row when
 secondary_ranges != 0

---
 src/lua/oltp_read_only.lua       |  8 ++++++--
 src/lua/oltp_read_write.lua      |  8 ++++++--
 src/lua/select_random_ranges.lua | 15 +++++++++++----
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/lua/oltp_read_only.lua b/src/lua/oltp_read_only.lua
index a54ed31b1..941c96fad 100755
--- a/src/lua/oltp_read_only.lua
+++ b/src/lua/oltp_read_only.lua
@@ -31,7 +31,9 @@ function prepare_statements()
 
    if sysbench.opt.range_selects then
       prepare_simple_ranges()
-      prepare_secondary_ranges()
+      if (sysbench.opt.secondary_ranges > 0) then
+         prepare_secondary_ranges()
+      end
       prepare_sum_ranges()
       prepare_order_ranges()
       prepare_distinct_ranges()
@@ -47,7 +49,9 @@ function event()
 
    if sysbench.opt.range_selects then
       execute_simple_ranges()
-      execute_secondary_ranges()
+      if (sysbench.opt.secondary_ranges > 0) then
+         execute_secondary_ranges()
+      end
       execute_sum_ranges()
       execute_order_ranges()
       execute_distinct_ranges()
diff --git a/src/lua/oltp_read_write.lua b/src/lua/oltp_read_write.lua
index 40ccbfe98..de9c4559c 100755
--- a/src/lua/oltp_read_write.lua
+++ b/src/lua/oltp_read_write.lua
@@ -31,7 +31,9 @@ function prepare_statements()
 
    if sysbench.opt.range_selects then
       prepare_simple_ranges()
-      prepare_secondary_ranges()
+      if (sysbench.opt.secondary_ranges > 0) then
+         prepare_secondary_ranges()
+      end
       prepare_sum_ranges()
       prepare_order_ranges()
       prepare_distinct_ranges()
@@ -51,7 +53,9 @@ function event()
 
    if sysbench.opt.range_selects then
       execute_simple_ranges()
-      execute_secondary_ranges()
+      if (sysbench.opt.secondary_ranges > 0) then
+         execute_secondary_ranges()
+      end
       execute_sum_ranges()
       execute_order_ranges()
       execute_distinct_ranges()
diff --git a/src/lua/select_random_ranges.lua b/src/lua/select_random_ranges.lua
index bea8b9741..9de96b818 100755
--- a/src/lua/select_random_ranges.lua
+++ b/src/lua/select_random_ranges.lua
@@ -39,10 +39,17 @@ function thread_init()
                              sysbench.opt.number_of_ranges - 1) ..
       "k BETWEEN ? AND ?"
 
-   stmt = con:prepare(string.format([[
-        SELECT count(k)
-          FROM sbtest1
-          WHERE %s]], ranges))
+   if (sysbench.opt.secondary_ranges == 0)
+      stmt = con:prepare(string.format([[
+         SELECT count(k)
+            FROM sbtest1
+            WHERE %s]], ranges))
+   else
+      stmt = con:prepare(string.format([[
+         SELECT sum(length(c))
+            FROM sbtest1
+            WHERE %s]], ranges))
+   end
 
    params = {}
    for j = 1, sysbench.opt.number_of_ranges*2 do

From 6f71b2550c8b0631ff54433fc269a8d48834bd7f Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Wed, 21 Sep 2022 17:42:48 +0800
Subject: [PATCH 08/27] remove ROW_FORMAT=COMPRESSED

---
 src/lua/oltp_common.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index bb589fd0b..63d2d40e9 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -204,7 +204,7 @@ function create_table(drv, con, table_num)
      c VARCHAR(512) DEFAULT '' NOT NULL,
      pad MEDIUMTEXT,
      %s (id)
-   ) %s %s ROW_FORMAT=COMPRESSED]],
+   ) %s %s]],
          table_num, id_def, id_index_def, engine_def,
          sysbench.opt.create_table_options)
    else

From f9458f4955b1d7f536338bebbda0933c8ccde667 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Wed, 21 Sep 2022 18:43:01 +0800
Subject: [PATCH 09/27] bugfix: add missing "then"

---
 src/lua/select_random_ranges.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lua/select_random_ranges.lua b/src/lua/select_random_ranges.lua
index 9de96b818..ab5afee15 100755
--- a/src/lua/select_random_ranges.lua
+++ b/src/lua/select_random_ranges.lua
@@ -39,7 +39,7 @@ function thread_init()
                              sysbench.opt.number_of_ranges - 1) ..
       "k BETWEEN ? AND ?"
 
-   if (sysbench.opt.secondary_ranges == 0)
+   if (sysbench.opt.secondary_ranges == 0) then
       stmt = con:prepare(string.format([[
          SELECT count(k)
             FROM sbtest1

From 53f7263c2b64aa4e7dbd90c57e9329ad7f844091 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Fri, 23 Sep 2022 15:20:56 +0800
Subject: [PATCH 10/27] execute_delete_inserts: fix for opt.use_file - fix

---
 src/lua/oltp_common.lua | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index 63d2d40e9..5cf7e44c8 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -302,7 +302,8 @@ local stmt_defs = {
       t.INT},
    inserts = {
       "INSERT INTO sbtest%u (id, k, c, pad) VALUES (?, ?, ?, ?)",
-      t.INT, t.INT, {t.CHAR, 120}, {t.CHAR, 60}},
+      t.INT, t.INT, {t.CHAR, 120},
+      {t.VARCHAR, 4194000}}, --lua bind VARCHAR can be larger than MySQL(65535)
 }
 
 function prepare_begin()
@@ -524,6 +525,7 @@ end
 
 function execute_delete_inserts()
    local tnum = get_table_num()
+   local file_c_val, file_pad_val
 
    for i = 1, sysbench.opt.delete_inserts do
       local id = get_id()
@@ -533,8 +535,14 @@ function execute_delete_inserts()
 
       param[tnum].inserts[1]:set(id)
       param[tnum].inserts[2]:set(k)
-      param[tnum].inserts[3]:set_rand_str(c_value_template)
-      param[tnum].inserts[4]:set_rand_str(pad_value_template)
+      if sysbench.opt.use_file then
+         file_c_val, file_pad_val = get_str_value()
+         param[tnum].inserts[3]:set(file_c_val)
+         param[tnum].inserts[4]:set(file_pad_val)
+      else
+         param[tnum].inserts[3]:set_rand_str(c_value_template)
+         param[tnum].inserts[4]:set_rand_str(pad_value_template)
+      end
 
       stmt[tnum].deletes:execute()
       stmt[tnum].inserts:execute()

From 82c151e8531a30a8c6285a9059abc997de8b80d7 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Fri, 23 Sep 2022 15:44:21 +0800
Subject: [PATCH 11/27] oltp_insert.lua: fix for opt.use_file

---
 src/lua/oltp_insert.lua | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/lua/oltp_insert.lua b/src/lua/oltp_insert.lua
index 3035d40b4..4e5d7e8e9 100755
--- a/src/lua/oltp_insert.lua
+++ b/src/lua/oltp_insert.lua
@@ -46,9 +46,15 @@ function event()
    local c_val = get_c_value()
    local pad_val = get_pad_value()
 
+   if sysbench.opt.use_file then
+      c_val, pad_val = get_str_value()
+   else
+      c_val = get_c_value()
+      pad_val = get_pad_value()
+   end
    if (drv:name() == "pgsql" and sysbench.opt.auto_inc) then
       con:query(string.format("INSERT INTO %s (k, c, pad) VALUES " ..
-                                 "(%d, '%s', '%s')",
+                                 "(%d, \"%s\", \"%s\")",
                               table_name, k_val, c_val, pad_val))
    else
       if (sysbench.opt.auto_inc) then
@@ -59,7 +65,7 @@ function event()
       end
 
       con:query(string.format("INSERT INTO %s (id, k, c, pad) VALUES " ..
-                                 "(%d, %d, '%s', '%s')",
+                                 "(%d, %d, \"%s\", \"%s\")",
                               table_name, i, k_val, c_val, pad_val))
    end
 

From 055e1887e586fce17de89182439069aa8968d26a Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Fri, 23 Sep 2022 15:45:20 +0800
Subject: [PATCH 12/27] oltp_insert.lua: fix for opt.use_file - 2

---
 src/lua/oltp_insert.lua | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/lua/oltp_insert.lua b/src/lua/oltp_insert.lua
index 4e5d7e8e9..e6e6e1039 100755
--- a/src/lua/oltp_insert.lua
+++ b/src/lua/oltp_insert.lua
@@ -43,8 +43,7 @@ end
 function event()
    local table_name = "sbtest" .. sysbench.rand.uniform(1, sysbench.opt.tables)
    local k_val = sysbench.rand.default(1, sysbench.opt.table_size)
-   local c_val = get_c_value()
-   local pad_val = get_pad_value()
+   local c_val, pad_val
 
    if sysbench.opt.use_file then
       c_val, pad_val = get_str_value()

From d7c64d501f3bef845e879ec97745214d2b0a09cc Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Fri, 23 Sep 2022 15:56:24 +0800
Subject: [PATCH 13/27] oltp_insert.lua: fix for opt.use_file - 3

---
 src/lua/oltp_common.lua | 3 ++-
 src/lua/oltp_insert.lua | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index 5cf7e44c8..f76dec97c 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -525,7 +525,8 @@ end
 
 function execute_delete_inserts()
    local tnum = get_table_num()
-   local file_c_val, file_pad_val
+   local file_c_val
+   local file_pad_val
 
    for i = 1, sysbench.opt.delete_inserts do
       local id = get_id()
diff --git a/src/lua/oltp_insert.lua b/src/lua/oltp_insert.lua
index e6e6e1039..16f46dd20 100755
--- a/src/lua/oltp_insert.lua
+++ b/src/lua/oltp_insert.lua
@@ -43,7 +43,8 @@ end
 function event()
    local table_name = "sbtest" .. sysbench.rand.uniform(1, sysbench.opt.tables)
    local k_val = sysbench.rand.default(1, sysbench.opt.table_size)
-   local c_val, pad_val
+   local c_val
+   local pad_val
 
    if sysbench.opt.use_file then
       c_val, pad_val = get_str_value()

From ef7fb9e80b9df363428aeb1e8fc157732ef367e4 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Fri, 23 Sep 2022 16:16:09 +0800
Subject: [PATCH 14/27] oltp_insert.lua: fix for opt.use_file : prevent string
 cutoff

---
 src/lua/oltp_insert.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lua/oltp_insert.lua b/src/lua/oltp_insert.lua
index 16f46dd20..dd0b92cf4 100755
--- a/src/lua/oltp_insert.lua
+++ b/src/lua/oltp_insert.lua
@@ -65,8 +65,8 @@ function event()
       end
 
       con:query(string.format("INSERT INTO %s (id, k, c, pad) VALUES " ..
-                                 "(%d, %d, \"%s\", \"%s\")",
-                              table_name, i, k_val, c_val, pad_val))
+                                 "(%d, %d, \"%s\", \"",
+                              table_name, i, k_val, c_val) ..  pad_val .. "\")")
    end
 
    check_reconnect()

From da9d074d7e450004470cec946a00605d56e7fe8c Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Fri, 23 Sep 2022 16:37:34 +0800
Subject: [PATCH 15/27] filestr: fix for multi thread

---
 src/lua/internal/sysbench.rand.lua | 5 ++---
 src/sb_rand.c                      | 6 ++++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/lua/internal/sysbench.rand.lua b/src/lua/internal/sysbench.rand.lua
index e339dfdbd..e333c0527 100644
--- a/src/lua/internal/sysbench.rand.lua
+++ b/src/lua/internal/sysbench.rand.lua
@@ -84,10 +84,9 @@ function sysbench.rand.uniform_double()
    return ffi.C.sb_rand_uniform_double()
 end
 
-buf1 = ffi.new("char[?]", 1024)
-buf2 = ffi.new("char[?]", 4194304)
-
 function sysbench.rand.filestr()
+   local buf1 = ffi.new("char[?]", 1024)
+   local buf2 = ffi.new("char[?]", 4194304)
    ffi.C.sb_file_str(buf1, buf2)
    return ffi.string(buf1), ffi.string(buf2)
 end
\ No newline at end of file
diff --git a/src/sb_rand.c b/src/sb_rand.c
index f67b59743..5090cc861 100644
--- a/src/sb_rand.c
+++ b/src/sb_rand.c
@@ -58,6 +58,7 @@
 #include "sb_logger.h"
 
 #include "sb_ck_pr.h"
+#include "pthread.h"
 
 TLS sb_rng_state_t sb_rng_state CK_CC_CACHELINE;
 
@@ -462,6 +463,7 @@ uint32_t sb_rand_zipfian(uint32_t a, uint32_t b)
 static FILE* file = NULL;
 static char* str_buf;
 static size_t len;
+static pthread_mutex_t file_mtx;
 
 int sb_file_init()
 {
@@ -473,6 +475,7 @@ int sb_file_init()
     log_text(LOG_FATAL, "Invalid filename: %s", sb_globals.filename);
     return 1;
   }
+  pthread_mutex_init(&file_mtx, NULL);
 
   len = 4194304 * sizeof(char);
   str_buf = (char *)malloc(len);
@@ -483,6 +486,7 @@ void sb_file_str(char* buf1, char* buf2)
 {
   assert(file != NULL);
 
+  pthread_mutex_lock(&file_mtx);
   ssize_t read = getline(&str_buf, &len, file);
   if (read != -1)
   {
@@ -502,6 +506,7 @@ void sb_file_str(char* buf1, char* buf2)
       memcpy(buf2, str_buf, read);
     }
   }
+  pthread_mutex_unlock(&file_mtx);
 }
 
 void sb_file_done()
@@ -515,6 +520,7 @@ void sb_file_done()
   {
     free(str_buf);
   }
+  pthread_mutex_destroy(&file_mtx);
 }
 
 

From 5b9dc3ce935cd3238c5471d68e3dd929ab56e1c3 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Fri, 23 Sep 2022 23:05:26 +0800
Subject: [PATCH 16/27] sb_file_str: use thread local

---
 src/lua/internal/sysbench.rand.lua | 15 ++++--
 src/sb_rand.c                      | 73 +++++++++++++++++-------------
 src/sb_rand.h                      |  2 +-
 3 files changed, 53 insertions(+), 37 deletions(-)

diff --git a/src/lua/internal/sysbench.rand.lua b/src/lua/internal/sysbench.rand.lua
index e333c0527..69451998d 100644
--- a/src/lua/internal/sysbench.rand.lua
+++ b/src/lua/internal/sysbench.rand.lua
@@ -33,7 +33,14 @@ uint32_t sb_rand_unique(void);
 void sb_rand_str(const char *, char *);
 uint32_t sb_rand_varstr(char *, uint32_t, uint32_t);
 double sb_rand_uniform_double(void);
-void sb_file_str(char *, char *);
+struct SbKeyVal {
+   size_t cap;
+   size_t klen;
+   size_t vlen;
+   char*  str; // also key
+   char*  val; // ptr after tab char
+};
+struct SbKeyVal* sb_file_str(void);
 ]]
 
 function sysbench.rand.uniform_uint64()
@@ -85,8 +92,6 @@ function sysbench.rand.uniform_double()
 end
 
 function sysbench.rand.filestr()
-   local buf1 = ffi.new("char[?]", 1024)
-   local buf2 = ffi.new("char[?]", 4194304)
-   ffi.C.sb_file_str(buf1, buf2)
-   return ffi.string(buf1), ffi.string(buf2)
+   local ms = ffi.C.sb_file_str()
+   return ffi.string(ms.str, ms.klen), ffi.string(ms.val, ms.vlen)
 end
\ No newline at end of file
diff --git a/src/sb_rand.c b/src/sb_rand.c
index 5090cc861..0446f8ce2 100644
--- a/src/sb_rand.c
+++ b/src/sb_rand.c
@@ -58,7 +58,7 @@
 #include "sb_logger.h"
 
 #include "sb_ck_pr.h"
-#include "pthread.h"
+#include <ctype.h>
 
 TLS sb_rng_state_t sb_rng_state CK_CC_CACHELINE;
 
@@ -460,10 +460,16 @@ uint32_t sb_rand_zipfian(uint32_t a, uint32_t b)
     sb_rand_zipfian_int(b - a + 1, zipf_exp, zipf_s, zipf_hIntegralX1) - 1;
 }
 
+struct SbKeyVal {
+  size_t cap;
+  size_t klen;
+  size_t vlen;
+  char*  str; // also key
+  char*  val; // ptr after tab char
+};
+static __thread struct SbKeyVal* g_kv = NULL;
+
 static FILE* file = NULL;
-static char* str_buf;
-static size_t len;
-static pthread_mutex_t file_mtx;
 
 int sb_file_init()
 {
@@ -475,38 +481,49 @@ int sb_file_init()
     log_text(LOG_FATAL, "Invalid filename: %s", sb_globals.filename);
     return 1;
   }
-  pthread_mutex_init(&file_mtx, NULL);
 
-  len = 4194304 * sizeof(char);
-  str_buf = (char *)malloc(len);
   return 0;
 }
 
-void sb_file_str(char* buf1, char* buf2)
+struct SbKeyVal* sb_file_str()
 {
   assert(file != NULL);
 
-  pthread_mutex_lock(&file_mtx);
-  ssize_t read = getline(&str_buf, &len, file);
-  if (read != -1)
-  {
-    int pos = 0;
-    while (pos < read) {
-      if (str_buf[pos] == '\t') {
-        memcpy(buf1, str_buf, pos);
-        buf1[pos] = 0;
-        memcpy(buf2, str_buf + pos + 1, read - pos - 2);
-        buf2[read - pos - 2] = 0;
-        break;
+  if (NULL == g_kv) {
+    g_kv = (struct SbKeyVal*)malloc(sizeof(struct SbKeyVal));
+    g_kv->klen = 0;
+    g_kv->vlen = 0;
+    g_kv->cap = 4*1024*1024;
+    g_kv->str = (char*)malloc(g_kv->cap);
+    g_kv->val = NULL;
+  }
+  while (true) {
+    ssize_t read = getline(&g_kv->str, &g_kv->cap, file);
+    if (read != -1) {
+      char* str = g_kv->str;
+      while (read > 0 && isspace((unsigned char)str[read])) {
+        str[--read] = '\0'; // trim trailing spaces
+      }
+      char* tab = (char*)memchr(str, '\t', read);
+      if (NULL == tab) {
+        g_kv->val = str + read; // empty c string
+        g_kv->vlen = 0;
+        g_kv->klen = read;
       }
-      pos++;
+      else {
+        tab[0] = '\0';
+        g_kv->klen = tab - str;
+        g_kv->vlen = read - (tab - str) - 1;
+        g_kv->val = tab + 1;
+      }
+      break;
     }
-
-    if (pos >= read) {
-      memcpy(buf2, str_buf, read);
+    else {
+      rewind(file);
+      fprintf(stderr, "sb_file_str: read eof, rewind(%s)\n", sb_globals.filename);
     }
   }
-  pthread_mutex_unlock(&file_mtx);
+  return g_kv;
 }
 
 void sb_file_done()
@@ -515,12 +532,6 @@ void sb_file_done()
   {
     fclose(file);
   }
-
-  if (str_buf)
-  {
-    free(str_buf);
-  }
-  pthread_mutex_destroy(&file_mtx);
 }
 
 
diff --git a/src/sb_rand.h b/src/sb_rand.h
index 3b88490cb..66073a277 100644
--- a/src/sb_rand.h
+++ b/src/sb_rand.h
@@ -72,7 +72,7 @@ void sb_rand_str(const char *, char *);
 uint32_t sb_rand_varstr(char *, uint32_t, uint32_t);
 
 int sb_file_init(void);
-void sb_file_str(char *, char *);
+struct SbKeyVal* sb_file_str(void);
 void sb_file_done(void);
 
 #endif /* SB_RAND_H */

From 0e9f0589aef7935e6057a16d4b8a95155083ffc0 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Wed, 28 Sep 2022 15:03:39 +0800
Subject: [PATCH 17/27] select_random_ranges.lua: use param "range_size" as cnt
 limit of fetch table when secondary_ranges == 2

---
 src/lua/select_random_ranges.lua | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/lua/select_random_ranges.lua b/src/lua/select_random_ranges.lua
index ab5afee15..14c6be9b0 100755
--- a/src/lua/select_random_ranges.lua
+++ b/src/lua/select_random_ranges.lua
@@ -44,6 +44,13 @@ function thread_init()
          SELECT count(k)
             FROM sbtest1
             WHERE %s]], ranges))
+   elseif (sysbench.opt.secondary_ranges == 2) then
+      -- MySQL doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery,
+      -- So we create an extra nested subquery
+      stmt = con:prepare(string.format([[
+         SELECT count(*), sum(length(c)) FROM sbtest1 WHERE id IN
+        (SELECT * FROM (SELECT id FROM sbtest1 WHERE %s LIMIT %d) as t)]],
+         ranges, sysbench.opt.range_size))
    else
       stmt = con:prepare(string.format([[
          SELECT sum(length(c))

From e9ea09fd064fb46070b9dd6fee74e86183f71ac1 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Mon, 10 Oct 2022 15:12:41 +0800
Subject: [PATCH 18/27] sql read stmt use length(c|pad) to avoid network
 bandwidth limit

---
 src/lua/oltp_common.lua          | 10 +++++-----
 src/lua/select_random_points.lua |  2 +-
 tests/t/script_oltp_help.t       |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index f76dec97c..63c539f31 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -274,22 +274,22 @@ end
 local t = sysbench.sql.type
 local stmt_defs = {
    point_selects = {
-      "SELECT c FROM sbtest%u WHERE id=?",
+      "SELECT length(c) FROM sbtest%u WHERE id=?",
       t.INT},
    simple_ranges = {
-      "SELECT c FROM sbtest%u WHERE id BETWEEN ? AND ?",
+      "SELECT length(c) FROM sbtest%u WHERE id BETWEEN ? AND ?",
       t.INT, t.INT},
    secondary_ranges = {
-      "SELECT c FROM sbtest%u WHERE k >= ? AND ? >=0 LIMIT %u",
+      "SELECT length(c) FROM sbtest%u WHERE k >= ? AND ? >=0 LIMIT %u",
       t.INT, t.INT},
    sum_ranges = {
       "SELECT SUM(k) FROM sbtest%u WHERE id BETWEEN ? AND ?",
        t.INT, t.INT},
    order_ranges = {
-      "SELECT c FROM sbtest%u WHERE id BETWEEN ? AND ? ORDER BY c",
+      "SELECT length(c) as lenc FROM sbtest%u WHERE id BETWEEN ? AND ? ORDER BY lenc",
        t.INT, t.INT},
    distinct_ranges = {
-      "SELECT DISTINCT c FROM sbtest%u WHERE id BETWEEN ? AND ? ORDER BY c",
+      "SELECT DISTINCT length(c) as lenc FROM sbtest%u WHERE id BETWEEN ? AND ? ORDER BY lenc",
       t.INT, t.INT},
    index_updates = {
       "UPDATE sbtest%u SET k=k+1 WHERE id=?",
diff --git a/src/lua/select_random_points.lua b/src/lua/select_random_points.lua
index 0afe2c941..2a65251d4 100755
--- a/src/lua/select_random_points.lua
+++ b/src/lua/select_random_points.lua
@@ -36,7 +36,7 @@ function thread_init()
    local points = string.rep("?, ", sysbench.opt.random_points - 1) .. "?"
 
    stmt = con:prepare(string.format([[
-        SELECT id, k, c, pad
+        SELECT id, k, length(c), length(pad)
           FROM sbtest1
           WHERE k IN (%s)
         ]], points))
diff --git a/tests/t/script_oltp_help.t b/tests/t/script_oltp_help.t
index 34be46602..fe622597c 100644
--- a/tests/t/script_oltp_help.t
+++ b/tests/t/script_oltp_help.t
@@ -14,7 +14,7 @@ OLTP usage information test
     --index_updates=N             Number of UPDATE index queries per transaction [1]
     --mysql_storage_engine=STRING Storage engine, if MySQL is used [innodb]
     --non_index_updates=N         Number of UPDATE non-index queries per transaction [1]
-    --secondary_ranges=N          Number of secondary range SELECT queries per transaction [1]
+    --secondary_ranges=[0|1|2]    Secondary index to fetch table SELECT queries per transaction [1]
     --order_ranges=N              Number of SELECT ORDER BY queries per transaction [1]
     --pgsql_variant=STRING        Use this PostgreSQL variant when running with the PostgreSQL driver. The only currently supported variant is 'redshift'. When enabled, create_secondary is automatically disabled, and delete_inserts is set to 0
     --point_selects=N             Number of point SELECT queries per transaction [10]

From 50e4dcb29d287703a9bb19e8db1a398061cf6913 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Tue, 17 Oct 2023 23:22:40 +0800
Subject: [PATCH 19/27] oltp_common.lua: add option start_id for parallel
 prepare one table

---
 src/lua/oltp_common.lua | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua
index 63c539f31..c4361be0e 100644
--- a/src/lua/oltp_common.lua
+++ b/src/lua/oltp_common.lua
@@ -33,6 +33,14 @@ end
 sysbench.cmdline.options = {
    table_size =
       {"Number of rows per table", 10000},
+   start_id =
+      {"Start id of rows for prepare insert into table, " ..
+       "used for parallel prepare data into one table, " ..
+       "users can run multiple sysbench process with different start_id concurrently. " ..
+       "in this case, range_size is the number of rows to be inserted, id of inserted " ..
+       "rows are {start_id, start_id + 1, ..., start_id + range_size - 1}. " ..
+       "this option should not be used with auto_inc. " ..
+       "default is 0 to conform origin sysbench", 0},
    range_size =
       {"Range size for range SELECT queries", 100},
    tables =
@@ -198,7 +206,7 @@ function create_table(drv, con, table_num)
 
    if sysbench.opt.use_file then
       query = string.format([[
-   CREATE TABLE sbtest%d(
+   CREATE TABLE IF NOT EXISTS sbtest%d(
      id %s,
      k INTEGER DEFAULT '0' NOT NULL,
      c VARCHAR(512) DEFAULT '' NOT NULL,
@@ -209,7 +217,7 @@ function create_table(drv, con, table_num)
          sysbench.opt.create_table_options)
    else
       query = string.format([[
-   CREATE TABLE sbtest%d(
+   CREATE TABLE IF NOT EXISTS sbtest%d(
      id %s,
      k INTEGER DEFAULT '0' NOT NULL,
      c CHAR(120) DEFAULT '' NOT NULL,
@@ -237,8 +245,18 @@ function create_table(drv, con, table_num)
 
    local c_val
    local pad_val
+   local start_id;
+   local finish_id;
 
-   for i = 1, sysbench.opt.table_size do
+   if sysbench.opt.start_id == 0 then
+      start_id = 1;
+      finish_id = sysbench.opt.table_size;
+   else
+      start_id = sysbench.opt.start_id;
+      finish_id = start_id + sysbench.opt.range_size - 1;
+   end
+
+   for i = start_id, finish_id do
 
       if sysbench.opt.use_file then
          c_val, pad_val = get_str_value()

From 0f10d07ec260376d448e8a93c43097e4a59c839d Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Sat, 28 Oct 2023 13:44:49 +0800
Subject: [PATCH 20/27] Add env MYSQL_CONN_CHARSET to set
 MYSQL_SET_CHARSET_NAME

---
 src/drivers/mysql/drv_mysql.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/drivers/mysql/drv_mysql.c b/src/drivers/mysql/drv_mysql.c
index 58600b8e0..177166553 100644
--- a/src/drivers/mysql/drv_mysql.c
+++ b/src/drivers/mysql/drv_mysql.c
@@ -411,6 +411,10 @@ static int mysql_drv_real_connect(db_mysql_conn_t *db_mysql_con)
     DEBUG("mysql_options(%p, %s, %s)",con, "MYSQL_OPT_COMPRESS", "NULL");
     mysql_options(con, MYSQL_OPT_COMPRESS, NULL);
   }
+  const char* cs = getenv("MYSQL_CONN_CHARSET");
+  if (cs) {
+    mysql_options(con, MYSQL_SET_CHARSET_NAME, cs);
+  }
 
   DEBUG("mysql_real_connect(%p, \"%s\", \"%s\", \"%s\", \"%s\", %u, \"%s\", %s)",
         con,

From 488a0dd905699f9a7331087398fdf4f1996f2c97 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Wed, 1 Nov 2023 23:17:05 +0800
Subject: [PATCH 21/27] select_random_ranges: add secondary_ranges == 3 for
 mysql mrr range_size rows

---
 src/lua/select_random_ranges.lua | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/lua/select_random_ranges.lua b/src/lua/select_random_ranges.lua
index 14c6be9b0..b72f82132 100755
--- a/src/lua/select_random_ranges.lua
+++ b/src/lua/select_random_ranges.lua
@@ -51,6 +51,14 @@ function thread_init()
          SELECT count(*), sum(length(c)) FROM sbtest1 WHERE id IN
         (SELECT * FROM (SELECT id FROM sbtest1 WHERE %s LIMIT %d) as t)]],
          ranges, sysbench.opt.range_size))
+   elseif (sysbench.opt.secondary_ranges == 3) then
+      -- MySQL does not generate MRR query plan for secondary_ranges == 2,
+      -- We add secondary_ranges == 3 as the query for get range_size rows
+      -- by MRR, secondary_ranges == 1 likely get more rows than range_size.
+      stmt = con:prepare(string.format([[
+         SELECT length(c)
+            FROM sbtest1
+            WHERE %s LIMIT %d]], ranges, sysbench.opt.range_size))
    else
       stmt = con:prepare(string.format([[
          SELECT sum(length(c))

From a7a03be4ff8a59b009c7ae55a594dc35c018f9c8 Mon Sep 17 00:00:00 2001
From: mytrygithub <30644711+mytrygithub@users.noreply.github.com>
Date: Tue, 7 Nov 2023 17:39:03 +0800
Subject: [PATCH 22/27] add index to table, change select-random-ranges select
 mulit db (#1)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add index to table, change select-random-ranges select mulit db

* change file to new dir

* Create ReadMe.md

* Update ReadMe.md

修改ReadMe格式

* cleanup code, add con:disconnect to thread_done

* Fixing syntax errors

* Update ReadMe.md

change parameters name
---
 multi_index_bench/ReadMe.md                |  26 +
 multi_index_bench/oltp_common.lua          | 610 +++++++++++++++++++++
 multi_index_bench/select_random_ranges.lua | 116 ++++
 3 files changed, 752 insertions(+)
 create mode 100644 multi_index_bench/ReadMe.md
 create mode 100755 multi_index_bench/oltp_common.lua
 create mode 100755 multi_index_bench/select_random_ranges.lua

diff --git a/multi_index_bench/ReadMe.md b/multi_index_bench/ReadMe.md
new file mode 100644
index 000000000..af3feae46
--- /dev/null
+++ b/multi_index_bench/ReadMe.md
@@ -0,0 +1,26 @@
+## 使用方法
+把该文件夹的两个文件放在执行sysbench命令的当前目录，sysbench会自动识别替换安装内容。
+
+## 主要修改内容：
+1、在数据表中添加索引数据项，用于测试带有索引的操作内容。
+
+2、select_random_ranges 可以测试多个库，同时完成随机选择数据库和索引。
+
+## 添加参数：
+```
+index_num =
+      {"create table index numbers", 0},
+table_with_index =
+      {"if index_nums > 0 create table with index", false},
+db_prefix =
+      {"manual create database prefix, this will disable mysql-db param", "sysbench"},
+db_num =
+      {"manual create database number, this will disable mysql-db param", 20}
+```
+
+## 用法：
+1、设置 index_num 为需要创建索引项的个数。table_with_index 表示是否在创建表是包含索引。
+
+2、select_random_ranges 需要使用的多个数据库需要通过 1，手动完成创建。（当前默认数据名是 sysbench1, sysbench2, sysbench3 ... ）
+
+
diff --git a/multi_index_bench/oltp_common.lua b/multi_index_bench/oltp_common.lua
new file mode 100755
index 000000000..e17cd5315
--- /dev/null
+++ b/multi_index_bench/oltp_common.lua
@@ -0,0 +1,610 @@
+-- Copyright (C) 2006-2018 Alexey Kopytov <akopytov@gmail.com>
+
+-- This program is free software; you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License as published by
+-- the Free Software Foundation; either version 2 of the License, or
+-- (at your option) any later version.
+
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+
+-- You should have received a copy of the GNU General Public License
+-- along with this program; if not, write to the Free Software
+-- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+-- -----------------------------------------------------------------------------
+-- Common code for OLTP benchmarks.
+-- -----------------------------------------------------------------------------
+
+function init()
+   assert(event ~= nil,
+          "this script is meant to be included by other OLTP scripts and " ..
+             "should not be called directly.")
+end
+
+if sysbench.cmdline.command == nil then
+   error("Command is required. Supported commands: prepare, warmup, run, " ..
+            "cleanup, help")
+end
+
+-- Command line options
+sysbench.cmdline.options = {
+   table_size =
+      {"Number of rows per table", 10000},
+   range_size =
+      {"Range size for range SELECT queries", 100},
+   tables =
+      {"Number of tables", 1},
+   point_selects =
+      {"Number of point SELECT queries per transaction", 10},
+   simple_ranges =
+      {"Number of simple range SELECT queries per transaction", 1},
+   secondary_ranges =
+      {"Number of secondary range SELECT queries per transaction", 1},
+   sum_ranges =
+      {"Number of SELECT SUM() queries per transaction", 1},
+   order_ranges =
+      {"Number of SELECT ORDER BY queries per transaction", 1},
+   distinct_ranges =
+      {"Number of SELECT DISTINCT queries per transaction", 1},
+   index_updates =
+      {"Number of UPDATE index queries per transaction", 1},
+   non_index_updates =
+      {"Number of UPDATE non-index queries per transaction", 1},
+   delete_inserts =
+      {"Number of DELETE/INSERT combinations per transaction", 1},
+   range_selects =
+      {"Enable/disable all range SELECT queries", true},
+   auto_inc =
+   {"Use AUTO_INCREMENT column as Primary Key (for MySQL), " ..
+       "or its alternatives in other DBMS. When disabled, use " ..
+       "client-generated IDs", true},
+   create_table_options =
+      {"Extra CREATE TABLE options", ""},
+   skip_trx =
+      {"Don't start explicit transactions and execute all queries " ..
+          "in the AUTOCOMMIT mode", false},
+   secondary =
+      {"Use a secondary index in place of the PRIMARY KEY", false},
+   create_secondary =
+      {"Create a secondary index in addition to the PRIMARY KEY", true},
+   reconnect =
+      {"Reconnect after every N events. The default (0) is to not reconnect",
+       0},
+   use_file =
+      {"If use a text file as dataset", false},
+   mysql_storage_engine =
+      {"Storage engine, if MySQL is used", "innodb"},
+   pgsql_variant =
+      {"Use this PostgreSQL variant when running with the " ..
+          "PostgreSQL driver. The only currently supported " ..
+          "variant is 'redshift'. When enabled, " ..
+          "create_secondary is automatically disabled, and " ..
+          "delete_inserts is set to 0"},
+   index_num =
+      {"create table index numbers", 0},
+   table_with_index =
+      {"if index_num > 0 create table with index", false},
+   db_prefix =
+      {"manual create database prefix, this will disable mysql-db param", "sysbench"},
+   db_num =
+      {"manual create database number, this will disable mysql-db param", 20}
+}
+
+-- Prepare the dataset. This command supports parallel execution, i.e. will
+-- benefit from executing with --threads > 1 as long as --tables > 1
+function cmd_prepare()
+   local drv = sysbench.sql.driver()
+   local con = drv:connect()
+
+   for i = sysbench.tid % sysbench.opt.threads + 1, sysbench.opt.tables,
+   sysbench.opt.threads do
+     create_table(drv, con, i)
+   end
+end
+
+-- Preload the dataset into the server cache. This command supports parallel
+-- execution, i.e. will benefit from executing with --threads > 1 as long as
+-- --tables > 1
+--
+-- PS. Currently, this command is only meaningful for MySQL/InnoDB benchmarks
+function cmd_warmup()
+   local drv = sysbench.sql.driver()
+   local con = drv:connect()
+
+   assert(drv:name() == "mysql", "warmup is currently MySQL only")
+
+   -- Do not create on disk tables for subsequent queries
+   con:query("SET tmp_table_size=2*1024*1024*1024")
+   con:query("SET max_heap_table_size=2*1024*1024*1024")
+
+   for i = sysbench.tid % sysbench.opt.threads + 1, sysbench.opt.tables,
+   sysbench.opt.threads do
+      local t = "sbtest" .. i
+      print("Preloading table " .. t)
+      con:query("ANALYZE TABLE sbtest" .. i)
+      con:query(string.format(
+                   "SELECT AVG(id) FROM " ..
+                      "(SELECT * FROM %s FORCE KEY (PRIMARY) " ..
+                      "LIMIT %u) t",
+                   t, sysbench.opt.table_size))
+      con:query(string.format(
+                   "SELECT COUNT(*) FROM " ..
+                      "(SELECT * FROM %s WHERE k LIKE '%%0%%' LIMIT %u) t",
+                   t, sysbench.opt.table_size))
+   end
+end
+
+-- Implement parallel prepare and warmup commands, define 'prewarm' as an alias
+-- for 'warmup'
+sysbench.cmdline.commands = {
+   prepare = {cmd_prepare, sysbench.cmdline.PARALLEL_COMMAND},
+   warmup = {cmd_warmup, sysbench.cmdline.PARALLEL_COMMAND},
+   prewarm = {cmd_warmup, sysbench.cmdline.PARALLEL_COMMAND}
+}
+
+
+-- Template strings of random digits with 11-digit groups separated by dashes
+
+-- 10 groups, 119 characters
+local c_value_template = "###########-###########-###########-" ..
+   "###########-###########-###########-" ..
+   "###########-###########-###########-" ..
+   "###########"
+
+-- 5 groups, 59 characters
+local pad_value_template = "###########-###########-###########-" ..
+   "###########-###########"
+
+function get_c_value()
+   return sysbench.rand.string(c_value_template)
+end
+
+function get_pad_value()
+   return sysbench.rand.string(pad_value_template)
+end
+
+function get_str_value()
+   return sysbench.rand.filestr()
+end
+
+function create_table(drv, con, table_num)
+   local id_index_def, id_def
+   local engine_def = ""
+   local query
+
+   if sysbench.opt.secondary then
+     id_index_def = "KEY xid"
+   else
+     id_index_def = "PRIMARY KEY"
+   end
+
+   if drv:name() == "mysql"
+   then
+      if sysbench.opt.auto_inc then
+         id_def = "INTEGER NOT NULL AUTO_INCREMENT"
+      else
+         id_def = "INTEGER NOT NULL"
+      end
+      engine_def = "/*! ENGINE = " .. sysbench.opt.mysql_storage_engine .. " */"
+   elseif drv:name() == "pgsql"
+   then
+      if not sysbench.opt.auto_inc then
+         id_def = "INTEGER NOT NULL"
+      elseif pgsql_variant == 'redshift' then
+        id_def = "INTEGER IDENTITY(1,1)"
+      else
+        id_def = "SERIAL"
+      end
+   else
+      error("Unsupported database driver:" .. drv:name())
+   end
+
+   print(string.format("Creating table 'sbtest%d'...", table_num))
+
+   local t_keys = {}
+   local t_keys_define = {}
+   local t_index = {}
+   for i = 1, sysbench.opt.index_num do
+      table.insert(t_keys, string.format([[k%d,]], i))
+      table.insert(t_keys_define, string.format([[k%d INTEGER DEFAULT '0' NOT NULL,]], i))
+      table.insert(t_index, string.format([[INDEX(k%d),]], i))
+   end
+   local string_keys=table.concat(t_keys)
+   local string_keys_define=table.concat(t_keys_define)
+   local string_index=""
+   if sysbench.opt.table_with_index then
+      string_index=table.concat(t_index)
+   end
+
+   if sysbench.opt.use_file then
+      query = string.format([[
+   CREATE TABLE sbtest%d(
+     id %s,
+     k INTEGER DEFAULT '0' NOT NULL,
+     %s
+     c VARBINARY(32000) DEFAULT '' NOT NULL,
+     pad MEDIUMBLOB,
+     %s
+     %s (id)
+   ) %s %s]],
+         table_num, id_def, string_keys_define, string_index, id_index_def, engine_def,
+         sysbench.opt.create_table_options)
+   else
+      query = string.format([[
+      CREATE TABLE sbtest%d(
+        id %s,
+        k INTEGER DEFAULT '0' NOT NULL,
+        %s
+        c BINARY(120) DEFAULT '' NOT NULL,
+        pad BINARY(60) DEFAULT '' NOT NULL,
+        %s (id)
+      ) %s %s]],
+         table_num, id_def, string_keys_define, id_index_def, engine_def,
+         sysbench.opt.create_table_options)
+   end
+
+   con:query(query)
+
+   if (sysbench.opt.table_size > 0) then
+      print(string.format("Inserting %d records into 'sbtest%d'",
+                          sysbench.opt.table_size, table_num))
+   end
+
+   if sysbench.opt.auto_inc then
+      query = "INSERT INTO sbtest" .. table_num .. "(k, " .. string_keys .. "c, pad) VALUES"
+   else
+      query = "INSERT INTO sbtest" .. table_num .. "(id, k, " .. string_keys .. "c, pad) VALUES"
+   end
+
+   con:bulk_insert_init(query)
+
+   local c_val
+   local pad_val
+
+   for i = 1, sysbench.opt.table_size do
+
+      if sysbench.opt.use_file then
+         c_val, pad_val = get_str_value()
+      else
+         c_val = get_c_value()
+         pad_val = get_pad_value()
+      end
+
+
+      local t_query = {}
+      for i = 1, sysbench.opt.index_num do
+         table.insert(t_query, string.format([[%d,]], sysbench.rand.default(1, sysbench.opt.table_size)))
+      end
+      string_query=table.concat(t_query, " ")
+
+      if (sysbench.opt.auto_inc) then
+         query = string.format("(%d, %s \"%s\", \"%s\")",
+                               sysbench.rand.default(1, sysbench.opt.table_size),
+                               string_query,
+                               c_val, pad_val)
+      else
+         query = string.format("(%d, %d, %s \"%s\", \"%s\")",
+                               i,
+                               sysbench.rand.default(1, sysbench.opt.table_size),
+                               string_query,
+                               c_val, pad_val)
+      end
+
+      con:bulk_insert_next(query)
+   end
+
+   con:bulk_insert_done()
+
+   if sysbench.opt.create_secondary then
+      print(string.format("Creating a secondary index on 'sbtest%d'...",
+                          table_num))
+      con:query(string.format("CREATE INDEX k_%d ON sbtest%d(k)",
+                              table_num, table_num))
+   end
+end
+
+local t = sysbench.sql.type
+local stmt_defs = {
+   point_selects = {
+      "SELECT length(c) FROM sbtest%u WHERE id=?",
+      t.INT},
+   simple_ranges = {
+      "SELECT length(c) FROM sbtest%u WHERE id BETWEEN ? AND ?",
+      t.INT, t.INT},
+   secondary_ranges = {
+      "SELECT length(c) FROM sbtest%u WHERE k >= ? AND ? >=0 LIMIT %u",
+      t.INT, t.INT},
+   sum_ranges = {
+      "SELECT SUM(k) FROM sbtest%u WHERE id BETWEEN ? AND ?",
+       t.INT, t.INT},
+   order_ranges = {
+      "SELECT length(c) as lenc FROM sbtest%u WHERE id BETWEEN ? AND ? ORDER BY lenc",
+       t.INT, t.INT},
+   distinct_ranges = {
+      "SELECT DISTINCT length(c) as lenc FROM sbtest%u WHERE id BETWEEN ? AND ? ORDER BY lenc",
+      t.INT, t.INT},
+   index_updates = {
+      "UPDATE sbtest%u SET k=k+1 WHERE id=?",
+      t.INT},
+   non_index_updates = {
+      "UPDATE sbtest%u SET c=? WHERE id=?",
+      {t.CHAR, 120}, t.INT},
+   deletes = {
+      "DELETE FROM sbtest%u WHERE id=?",
+      t.INT},
+   inserts = {
+      "INSERT INTO sbtest%u (id, k, c, pad) VALUES (?, ?, ?, ?)",
+      t.INT, t.INT, {t.CHAR, 120},
+      {t.VARCHAR, 4194000}}, --lua bind VARCHAR can be larger than MySQL(65535)
+}
+
+function prepare_begin()
+   stmt.begin = con:prepare("BEGIN")
+end
+
+function prepare_commit()
+   stmt.commit = con:prepare("COMMIT")
+end
+
+function prepare_for_each_table(key)
+   for t = 1, sysbench.opt.tables do
+      if key == "secondary_ranges" then
+         stmt[t][key] = con:prepare(string.format(stmt_defs[key][1], t, sysbench.opt.range_size))
+      else
+         stmt[t][key] = con:prepare(string.format(stmt_defs[key][1], t))
+      end
+
+      local nparam = #stmt_defs[key] - 1
+
+      if nparam > 0 then
+         param[t][key] = {}
+      end
+
+      for p = 1, nparam do
+         local btype = stmt_defs[key][p+1]
+         local len
+
+         if type(btype) == "table" then
+            len = btype[2]
+            btype = btype[1]
+         end
+         if btype == sysbench.sql.type.VARCHAR or
+            btype == sysbench.sql.type.CHAR then
+               param[t][key][p] = stmt[t][key]:bind_create(btype, len)
+         else
+            param[t][key][p] = stmt[t][key]:bind_create(btype)
+         end
+      end
+
+      if nparam > 0 then
+         stmt[t][key]:bind_param(unpack(param[t][key]))
+      end
+   end
+end
+
+function prepare_point_selects()
+   prepare_for_each_table("point_selects")
+end
+
+function prepare_simple_ranges()
+   prepare_for_each_table("simple_ranges")
+end
+
+function prepare_secondary_ranges()
+   prepare_for_each_table("secondary_ranges")
+end
+
+function prepare_sum_ranges()
+   prepare_for_each_table("sum_ranges")
+end
+
+function prepare_order_ranges()
+   prepare_for_each_table("order_ranges")
+end
+
+function prepare_distinct_ranges()
+   prepare_for_each_table("distinct_ranges")
+end
+
+function prepare_index_updates()
+   prepare_for_each_table("index_updates")
+end
+
+function prepare_non_index_updates()
+   prepare_for_each_table("non_index_updates")
+end
+
+function prepare_delete_inserts()
+   prepare_for_each_table("deletes")
+   prepare_for_each_table("inserts")
+end
+
+function thread_init()
+   drv = sysbench.sql.driver()
+   con = drv:connect()
+
+   -- Create global nested tables for prepared statements and their
+   -- parameters. We need a statement and a parameter set for each combination
+   -- of connection/table/query
+   stmt = {}
+   param = {}
+
+   for t = 1, sysbench.opt.tables do
+      stmt[t] = {}
+      param[t] = {}
+   end
+
+   -- This function is a 'callback' defined by individual benchmark scripts
+   prepare_statements()
+end
+
+-- Close prepared statements
+function close_statements()
+   for t = 1, sysbench.opt.tables do
+      for k, s in pairs(stmt[t]) do
+         stmt[t][k]:close()
+      end
+   end
+   if (stmt.begin ~= nil) then
+      stmt.begin:close()
+   end
+   if (stmt.commit ~= nil) then
+      stmt.commit:close()
+   end
+end
+
+function thread_done()
+   close_statements()
+   con:disconnect()
+end
+
+function cleanup()
+   local drv = sysbench.sql.driver()
+   local con = drv:connect()
+
+   for i = 1, sysbench.opt.tables do
+      print(string.format("Dropping table 'sbtest%d'...", i))
+      con:query("DROP TABLE IF EXISTS sbtest" .. i )
+   end
+end
+
+local function get_table_num()
+   return sysbench.rand.uniform(1, sysbench.opt.tables)
+end
+
+local function get_id()
+   return sysbench.rand.default(1, sysbench.opt.table_size)
+end
+
+function begin()
+   stmt.begin:execute()
+end
+
+function commit()
+   stmt.commit:execute()
+end
+
+function execute_point_selects()
+   local tnum = get_table_num()
+   local i
+
+   for i = 1, sysbench.opt.point_selects do
+      param[tnum].point_selects[1]:set(get_id())
+
+      stmt[tnum].point_selects:execute()
+   end
+end
+
+local function execute_range(key)
+   local tnum = get_table_num()
+
+   for i = 1, sysbench.opt[key] do
+      local id = get_id()
+
+      param[tnum][key][1]:set(id)
+      param[tnum][key][2]:set(id + sysbench.opt.range_size - 1)
+
+      stmt[tnum][key]:execute()
+   end
+end
+
+function execute_simple_ranges()
+   execute_range("simple_ranges")
+end
+
+function execute_secondary_ranges()
+   execute_range("secondary_ranges")
+end
+
+function execute_sum_ranges()
+   execute_range("sum_ranges")
+end
+
+function execute_order_ranges()
+   execute_range("order_ranges")
+end
+
+function execute_distinct_ranges()
+   execute_range("distinct_ranges")
+end
+
+function execute_index_updates()
+   local tnum = get_table_num()
+
+   for i = 1, sysbench.opt.index_updates do
+      param[tnum].index_updates[1]:set(get_id())
+
+      stmt[tnum].index_updates:execute()
+   end
+end
+
+function execute_non_index_updates()
+   local tnum = get_table_num()
+   local c_val;
+
+   for i = 1, sysbench.opt.non_index_updates do
+      if sysbench.opt.use_file then
+         c_val = get_str_value()
+      else
+         c_val = get_c_value()
+      end
+      param[tnum].non_index_updates[1]:set(c_val)
+      param[tnum].non_index_updates[2]:set(get_id())
+
+      stmt[tnum].non_index_updates:execute()
+   end
+end
+
+function execute_delete_inserts()
+   local tnum = get_table_num()
+   local file_c_val
+   local file_pad_val
+
+   for i = 1, sysbench.opt.delete_inserts do
+      local id = get_id()
+      local k = get_id()
+
+      param[tnum].deletes[1]:set(id)
+
+      param[tnum].inserts[1]:set(id)
+      param[tnum].inserts[2]:set(k)
+      if sysbench.opt.use_file then
+         file_c_val, file_pad_val = get_str_value()
+         param[tnum].inserts[3]:set(file_c_val)
+         param[tnum].inserts[4]:set(file_pad_val)
+      else
+         param[tnum].inserts[3]:set_rand_str(c_value_template)
+         param[tnum].inserts[4]:set_rand_str(pad_value_template)
+      end
+
+      stmt[tnum].deletes:execute()
+      stmt[tnum].inserts:execute()
+   end
+end
+
+-- Re-prepare statements if we have reconnected, which is possible when some of
+-- the listed error codes are in the --mysql-ignore-errors list
+function sysbench.hooks.before_restart_event(errdesc)
+   if errdesc.sql_errno == 2013 or -- CR_SERVER_LOST
+      errdesc.sql_errno == 2055 or -- CR_SERVER_LOST_EXTENDED
+      errdesc.sql_errno == 2006 or -- CR_SERVER_GONE_ERROR
+      errdesc.sql_errno == 2011    -- CR_TCP_CONNECTION
+   then
+      close_statements()
+      prepare_statements()
+   end
+end
+
+function check_reconnect()
+   if sysbench.opt.reconnect > 0 then
+      transactions = (transactions or 0) + 1
+      if transactions % sysbench.opt.reconnect == 0 then
+         close_statements()
+         con:reconnect()
+         prepare_statements()
+      end
+   end
+end
diff --git a/multi_index_bench/select_random_ranges.lua b/multi_index_bench/select_random_ranges.lua
new file mode 100755
index 000000000..3d062e7f9
--- /dev/null
+++ b/multi_index_bench/select_random_ranges.lua
@@ -0,0 +1,116 @@
+#!/usr/bin/env sysbench
+-- This test is designed for testing MariaDB's key_cache_segments for MyISAM,
+-- and should work with other storage engines as well.
+--
+-- For details about key_cache_segments please refer to:
+-- http://kb.askmonty.org/v/segmented-key-cache
+--
+
+require("oltp_common")
+
+-- Add --number-of-ranges and --delta to the list of standard OLTP options
+sysbench.cmdline.options.number_of_ranges =
+   {"Number of random BETWEEN ranges per SELECT", 10}
+sysbench.cmdline.options.delta =
+   {"Size of BETWEEN ranges", 5}
+
+-- Override standard prepare/cleanup OLTP functions, as this benchmark does not
+-- support multiple tables
+oltp_prepare = prepare
+oltp_cleanup = cleanup
+
+function prepare()
+   assert(sysbench.opt.tables == 1, "this benchmark does not support " ..
+             "--tables > 1")
+   oltp_prepare()
+end
+
+function cleanup()
+   assert(sysbench.opt.tables == 1, "this benchmark does not support " ..
+             "--tables > 1")
+   oltp_cleanup()
+end
+
+function thread_init()
+   stmt = {}
+   params = {}
+   drv = sysbench.sql.driver()
+   con = drv:connect()
+   for db_id = 1, sysbench.opt.db_num do
+      local db_name = string.format("%s%d",sysbench.opt.db_prefix, db_id)
+      stmt[db_id] = {}
+      params[db_id] = {}
+      for kid = 1, sysbench.opt.index_num do
+         stmt[db_id][kid] = {}
+         local k_name = string.format("k%d",kid)
+         local ranges = string.rep(k_name .. " BETWEEN ? AND ? OR ",
+                                 sysbench.opt.number_of_ranges - 1) ..
+            k_name .. " BETWEEN ? AND ?"
+
+         if (sysbench.opt.secondary_ranges == 0) then
+            stmt[db_id][kid] = con:prepare(string.format([[
+               SELECT count(k)
+               FROM %s.sbtest1
+               WHERE %s]], db_name, ranges))
+         elseif (sysbench.opt.secondary_ranges == 2) then
+            -- MySQL doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery,
+            -- So we create an extra nested subquery
+            stmt[db_id][kid] = con:prepare(string.format([[
+               SELECT count(*), sum(length(c)) FROM %s.sbtest1 WHERE id IN
+               (SELECT * FROM (SELECT id FROM %s.sbtest1 WHERE %s LIMIT %d) as t)]],
+               db_name, db_name, ranges, sysbench.opt.range_size))
+         elseif (sysbench.opt.secondary_ranges == 3) then
+            -- MySQL does not generate MRR query plan for secondary_ranges == 2,
+            -- We add secondary_ranges == 3 as the query for get range_size rows
+            -- by MRR, secondary_ranges == 1 likely get more rows than range_size.
+            stmt[db_id][kid] = con:prepare(string.format([[
+               SELECT length(c)
+               FROM %s.sbtest1
+               WHERE %s LIMIT %d]], db_name, ranges, sysbench.opt.range_size))
+         else
+            stmt[db_id][kid] = con:prepare(string.format([[
+               SELECT sum(length(c))
+               FROM %s.sbtest1
+               WHERE %s]], db_name, ranges))
+         end
+         params[db_id][kid] = {}
+         for j = 1, sysbench.opt.number_of_ranges*2 do
+            params[db_id][kid][j] = stmt[db_id][kid]:bind_create(sysbench.sql.type.INT)
+         end
+
+         stmt[db_id][kid]:bind_param(unpack(params[db_id][kid]))
+      end
+   end
+
+   rlen = sysbench.opt.table_size / sysbench.opt.threads
+
+   thread_id = sysbench.tid % sysbench.opt.threads
+end
+
+function thread_done()
+   for db_id = 1, sysbench.opt.db_num do
+      for kid = 1, sysbench.opt.index_num do
+         stmt[db_id][kid]:close()
+      end
+   end
+   con:disconnect()
+end
+
+function event()
+   -- To prevent overlapping of our range queries we need to partition the whole
+   -- table into 'threads' segments and then make each thread work with its
+   -- own segment.
+   local lkid = sysbench.rand.default(1, sysbench.opt.index_num)
+   local db_id = sysbench.rand.default(1, sysbench.opt.db_num)
+   for i = 1, sysbench.opt.number_of_ranges*2, 2 do
+      local rmin = rlen * thread_id
+      local rmax = rmin + rlen
+      local val = sysbench.rand.default(rmin, rmax)
+      params[db_id][lkid][i]:set(val)
+      params[db_id][lkid][i+1]:set(val + sysbench.opt.delta)
+   end
+
+   stmt[db_id][lkid]:execute()
+
+   check_reconnect()
+end

From e07412b602bfa878849cbbacea15cbd84e18ce16 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Tue, 7 Nov 2023 17:46:35 +0800
Subject: [PATCH 23/27] git mv ReadMe.md README.md

---
 multi_index_bench/{ReadMe.md => README.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename multi_index_bench/{ReadMe.md => README.md} (100%)

diff --git a/multi_index_bench/ReadMe.md b/multi_index_bench/README.md
similarity index 100%
rename from multi_index_bench/ReadMe.md
rename to multi_index_bench/README.md

From 8250919d398fa3e73a1bad4c0598e9d4f6b9bec3 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Tue, 7 Nov 2023 17:45:55 +0800
Subject: [PATCH 24/27] Update multi_index_bench/README.md

---
 multi_index_bench/README.md | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/multi_index_bench/README.md b/multi_index_bench/README.md
index af3feae46..8b729cb65 100644
--- a/multi_index_bench/README.md
+++ b/multi_index_bench/README.md
@@ -2,16 +2,15 @@
 把该文件夹的两个文件放在执行sysbench命令的当前目录，sysbench会自动识别替换安装内容。
 
 ## 主要修改内容：
-1、在数据表中添加索引数据项，用于测试带有索引的操作内容。
-
-2、select_random_ranges 可以测试多个库，同时完成随机选择数据库和索引。
+1. 在数据表中添加索引数据项，用于测试带有索引的操作内容。
+2. select_random_ranges 可以测试多个库，同时完成随机选择数据库和索引。
 
 ## 添加参数：
 ```
 index_num =
-      {"create table index numbers", 0},
+      {"number of indexes, such number of integer columns will be created for those secondary indexes", 0},
 table_with_index =
-      {"if index_nums > 0 create table with index", false},
+      {"if index_nums > 0, indexes will be created at table creation", false},
 db_prefix =
       {"manual create database prefix, this will disable mysql-db param", "sysbench"},
 db_num =
@@ -19,8 +18,7 @@ db_num =
 ```
 
 ## 用法：
-1、设置 index_num 为需要创建索引项的个数。table_with_index 表示是否在创建表是包含索引。
-
-2、select_random_ranges 需要使用的多个数据库需要通过 1，手动完成创建。（当前默认数据名是 sysbench1, sysbench2, sysbench3 ... ）
+1. 设置 index_num 为需要创建的次级索引的个数，这会创建相同数量的额外整数列(k_1, k_2, ...)。table_with_index 表示是否在创建表是包含索引。
+2. select_random_ranges 需要使用的多个数据库需要通过 1，手动完成创建。（当前默认数据名是 sysbench1, sysbench2, sysbench3 ... ）
 
 

From c48345fa134c458fd9862f10692c992490df35dd Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Wed, 24 Jan 2024 19:17:38 +0800
Subject: [PATCH 25/27] Add env var ROWS_BEFORE_COMMIT (default = 1000).

---
 src/db_driver.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/db_driver.c b/src/db_driver.c
index c0b07b19d..bfcf7f13a 100644
--- a/src/db_driver.c
+++ b/src/db_driver.c
@@ -43,7 +43,7 @@
 #define BULK_PACKET_SIZE (5120*1024)
 
 /* How many rows to insert before COMMITs (used in bulk insert) */
-#define ROWS_BEFORE_COMMIT 1000
+static int ROWS_BEFORE_COMMIT = 1000;
 
 /* Global variables */
 db_globals_t db_globals CK_CC_CACHELINE;
@@ -131,7 +131,7 @@ void db_print_help(void)
   log_text(LOG_NOTICE, "General database options:\n");
   sb_print_options(db_args);
   log_text(LOG_NOTICE, "");
-  
+
   log_text(LOG_NOTICE, "Compiled-in database drivers:");
   SB_LIST_FOR_EACH(pos, &drivers)
   {
@@ -841,7 +841,7 @@ int db_parse_arguments(void)
   db_globals.driver = sb_get_value_string("db-driver");
 
   db_globals.debug = sb_get_value_flag("db-debug");
-  
+
   return 0;
 }
 
@@ -859,7 +859,7 @@ int db_print_value(db_bind_t *var, char *buf, int buflen)
     n = snprintf(buf, buflen, "NULL");
     return (n < buflen) ? n : -1;
   }
-  
+
   switch (var->type) {
     case DB_TYPE_TINYINT:
       n = snprintf(buf, buflen, "%hhd", *(char *)var->buffer);
@@ -955,7 +955,9 @@ int db_bulk_insert_init(db_conn_t *con, const char *query, size_t query_len)
   con->bulk_buffer = (char *)malloc(con->bulk_buflen);
   if (con->bulk_buffer == NULL)
     return 1;
-  
+
+  ROWS_BEFORE_COMMIT = atoi(getenv("ROWS_BEFORE_COMMIT")?:"1000");
+
   con->bulk_commit_max = driver_caps.needs_commit ? ROWS_BEFORE_COMMIT : 0;
   con->bulk_commit_cnt = 0;
   strcpy(con->bulk_buffer, query);

From ca538b47f507923fb878edf13ecfa5857daca410 Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Thu, 25 Jan 2024 14:56:37 +0800
Subject: [PATCH 26/27] Add env BULK_PACKET_SIZE (default 5242880 ( =
 5120*1024))

---
 src/db_driver.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/db_driver.c b/src/db_driver.c
index bfcf7f13a..7feda408d 100644
--- a/src/db_driver.c
+++ b/src/db_driver.c
@@ -40,7 +40,7 @@
 #include "sb_ck_pr.h"
 
 /* Query length limit for bulk insert queries */
-#define BULK_PACKET_SIZE (5120*1024)
+static unsigned int BULK_PACKET_SIZE = 5120*1024;
 
 /* How many rows to insert before COMMITs (used in bulk insert) */
 static int ROWS_BEFORE_COMMIT = 1000;
@@ -943,6 +943,8 @@ int db_bulk_insert_init(db_conn_t *con, const char *query, size_t query_len)
     return 1;
   }
 
+  BULK_PACKET_SIZE = atoi(getenv("BULK_PACKET_SIZE")?:"5242880");
+
   /* Allocate query buffer */
   if (query_len + 1 > BULK_PACKET_SIZE)
   {

From eea572ff628685037a49217d8d36a2d33fe71fcd Mon Sep 17 00:00:00 2001
From: leipeng <peng@topling.cn>
Date: Mon, 5 Feb 2024 10:08:07 +0800
Subject: [PATCH 27/27] Add env MAX_BULK_CNT(default = 1000)

---
 src/db_driver.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/db_driver.c b/src/db_driver.c
index 7feda408d..db161ecd7 100644
--- a/src/db_driver.c
+++ b/src/db_driver.c
@@ -41,6 +41,7 @@
 
 /* Query length limit for bulk insert queries */
 static unsigned int BULK_PACKET_SIZE = 5120*1024;
+static unsigned int MAX_BULK_CNT = 1000;
 
 /* How many rows to insert before COMMITs (used in bulk insert) */
 static int ROWS_BEFORE_COMMIT = 1000;
@@ -944,6 +945,7 @@ int db_bulk_insert_init(db_conn_t *con, const char *query, size_t query_len)
   }
 
   BULK_PACKET_SIZE = atoi(getenv("BULK_PACKET_SIZE")?:"5242880");
+  MAX_BULK_CNT = atoi(getenv("MAX_BULK_CNT")?:"1000");
 
   /* Allocate query buffer */
   if (query_len + 1 > BULK_PACKET_SIZE)
@@ -997,7 +999,8 @@ int db_bulk_insert_next(db_conn_t *con, const char *query, size_t query_len)
     Reserve space for '\0' and ',' (if not the first chunk in
     a bulk insert
   */
-  if (con->bulk_ptr + query_len + 1 + (con->bulk_cnt>0) > con->bulk_buflen)
+  if (con->bulk_cnt >= MAX_BULK_CNT ||
+      con->bulk_ptr + query_len + 1 + (con->bulk_cnt>0) > con->bulk_buflen)
   {
     /* Is this a first row? */
     if (!con->bulk_cnt)