Skip to content

Commit 7e6b839

Browse files
PavelShilin89PavelShilin89
authored
Merge cb5b57e into e8a7131 
2 parents e8a7131 + cb5b57e commit 7e6b839

File tree

4 files changed

+198
-60
lines changed

4 files changed

+198
-60
lines changed

‎test/clt-tests/base/replication/join-cluster-on-all-nodes.recb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
––– input –––
2-
for n in `seq 2 $INSTANCE`; do mysql -h0 -P${n}306 -e "join cluster ${CLUSTER_NAME} at '127.0.0.1:1312'"; done;
2+
for n in `seq 2 $INSTANCE`; do mysql -h0 -P${n}306 -e "join cluster ${CLUSTER_NAME} at '127.0.0.1:$((n*1000+312))'"; done
33
––– output –––
44
––– input –––
55
mysql -h0 -P${INSTANCE}306 -e "show status like 'cluster_${CLUSTER_NAME}_status'\G"

‎test/clt-tests/base/searchd-with-flexible-ports.conf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ searchd {
1414
data_dir = /var/log/manticore-${INSTANCE}
1515
query_log_format = sphinxql
1616
query_log_commands = 1
17+
server_id = ${INSTANCE}
1718
}
1819
1920
EOF

‎test/clt-tests/core/test-freeze-unfreeze-update.rec

Lines changed: 131 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -9,31 +9,12 @@ if timeout 10 grep -qm1 '\[BUDDY\] started' <(tail -n 1000 -f /tmp/searchd.log);
99
––– output –––
1010
Buddy started!
1111
––– input –––
12-
mysql -P9315 -h0 -e "drop table if exists t; create table t(s string); insert into t values(1, 'a'); flush ramchunk t"; echo $?
12+
mysql -P9315 -h0 -e "unfreeze t; drop table if exists t; create table t(a string); insert into t values(1, 'a'); flush ramchunk t; freeze t; update t set a='b' where id = 1;" &
1313
––– output –––
14-
0
15-
––– input –––
16-
mysql -P9315 -h0 -e "select * from t;"
17-
––– output –––
18-
+------+------+
19-
| id | s |
20-
+------+------+
21-
| 1 | a |
22-
+------+------+
14+
[%{NUMBER}] %{NUMBER}
2315
––– input –––
2416
(rm -f /tmp/update.sql; for n in `seq 1 10000`; do echo "update t set s='b' where id=1;" >> /tmp/update.sql; done; nohup sh -c "while true; do mysql -P9315 -h0 < /tmp/update.sql | break; done" &>/dev/null &) 2>/dev/null
2517
––– output –––
26-
––– input –––
27-
sleep 30; mysql -P9315 -h0 -e "select * from t;"
28-
––– output –––
29-
+------+------+
30-
| id | s |
31-
+------+------+
32-
| 1 | b |
33-
+------+------+
34-
––– input –––
35-
stdbuf -oL mysql -P9315 -h0 -e "freeze t"
36-
––– output –––
3718
+------------------------+------------------------+
3819
| file | normalized |
3920
+------------------------+------------------------+
@@ -49,23 +30,140 @@ stdbuf -oL mysql -P9315 -h0 -e "freeze t"
4930
| /tmp/data/t/t.0.spp | /tmp/data/t/t.0.spp |
5031
| /tmp/data/t/t.0.spt | /tmp/data/t/t.0.spt |
5132
| /tmp/data/t/t.meta | /tmp/data/t/t.meta |
52-
| /tmp/data/t/t.ram | /tmp/data/t/t.ram |
5333
| /tmp/data/t/t.settings | /tmp/data/t/t.settings |
5434
+------------------------+------------------------+
5535
––– input –––
56-
mysql -P9315 -h0 -e "unfreeze t"; echo $?
36+
export INSTANCE=1
37+
––– output –––
38+
––– block: ../base/replication/start-searchd-precach –––
39+
––– input –––
40+
export INSTANCE=2
41+
––– output –––
42+
––– block: ../base/replication/start-searchd-precach –––
43+
––– input –––
44+
export INSTANCE=3
45+
––– output –––
46+
––– block: ../base/replication/start-searchd-precach –––
47+
––– input –––
48+
export CLUSTER_NAME=replication
49+
––– output –––
50+
––– input –––
51+
mysql -h0 -P1306 -e "create cluster ${CLUSTER_NAME}"
52+
––– output –––
53+
––– input –––
54+
mysql -h0 -P1306 -e "show status like 'cluster_${CLUSTER_NAME}_status'\G"
55+
––– output –––
56+
*************************** 1. row ***************************
57+
Counter: cluster_#!/[a-z]+/!#_status
58+
Value: primary
59+
––– input –––
60+
for n in `seq 2 $INSTANCE`; do mysql -h0 -P${n}306 -e "join cluster ${CLUSTER_NAME} at '127.0.0.1:1312'"; done;
61+
––– output –––
62+
––– input –––
63+
mysql -h0 -P1306 -e "SHOW STATUS LIKE 'cluster_replication_status'\G"
64+
––– output –––
65+
*************************** 1. row ***************************
66+
Counter: cluster_replication_status
67+
Value: primary
68+
––– input –––
69+
mysql -h0 -P2306 -e "SHOW STATUS LIKE 'cluster_replication_status'\G"
70+
––– output –––
71+
*************************** 1. row ***************************
72+
Counter: cluster_replication_status
73+
Value: primary
74+
––– input –––
75+
mysql -h0 -P3306 -e "SHOW STATUS LIKE 'cluster_replication_status'\G"
76+
––– output –––
77+
*************************** 1. row ***************************
78+
Counter: cluster_replication_status
79+
Value: primary
80+
––– input –––
81+
php -d memory_limit=2G ./test/clt-tests/scripts/load_names_attr.php --batch-size=100000 --concurrency=4 --docs=1000000 --start-id=1 --port=2306 > /dev/null
82+
––– output –––
83+
––– input –––
84+
mysql -h0 -P2306 -e "SELECT * FROM name ORDER BY id ASC LIMIT 1000000 OPTION max_matches=1000000;" > /tmp/name_data.txt
85+
––– output –––
86+
––– input –––
87+
md5sum /tmp/name_data.txt
88+
––– output –––
89+
85c098de3fcf2065d57d61f47ef68fae /tmp/name_data.txt
90+
––– input –––
91+
mysql -h0 -P2306 -e "SHOW TABLES"
92+
––– output –––
93+
+-------+------+
94+
| Table | Type |
95+
+-------+------+
96+
| name | rt |
97+
+-------+------+
98+
––– input –––
99+
mysql -h0 -P2306 -e "ALTER CLUSTER replication ADD name;"
100+
––– output –––
101+
––– input –––
102+
mysql -P1306 -h0 -e "SELECT COUNT(*) FROM name;"
103+
––– output –––
104+
+----------+
105+
| count(*) |
106+
+----------+
107+
| 1000000 |
108+
+----------+
109+
––– input –––
110+
mysql -P1306 -h0 -e "DESCRIBE name;"
111+
––– output –––
112+
+-----------------+--------------+----------------+
113+
| Field | Type | Properties |
114+
+-----------------+--------------+----------------+
115+
| id | bigint | |
116+
| username | text | indexed stored |
117+
| gender | string | |
118+
| age | uint | |
119+
| salary | float | |
120+
| discount | float | |
121+
| is_active | bool | |
122+
| last_login | timestamp | |
123+
| product_codes | mva | |
124+
| large_values | mva64 | |
125+
| additional_info | json | |
126+
| location_vector | float_vector | knn |
127+
+-----------------+--------------+----------------+
128+
––– input –––
129+
mysql -P1306 -h0 -e "SELECT * FROM name ORDER BY id ASC LIMIT 10;"
130+
––– output –––
131+
+------+-----------------------+--------+------+---------------+-----------+-----------+------------+---------------+-----------------+---------------------------------------+------------------------------------------+
132+
| id | username | gender | age | salary | discount | is_active | last_login | product_codes | large_values | additional_info | location_vector |
133+
+------+-----------------------+--------+------+---------------+-----------+-----------+------------+---------------+-----------------+---------------------------------------+------------------------------------------+
134+
| 1 | JOHN JOHNSON | male | 24 | 38055.761719 | 46.259998 | 1 | 1697454371 | 924,950 | 4991378,7837113 | {"city":"New York","hobby":"reading"} | 90.099998,-100.690002,0.737000,0.125000 |
135+
| 2 | ROBERT WILLIAMS | female | 25 | 44777.058594 | 25.760000 | 1 | 1699072085 | 113,999 | 3989541,4344769 | {"city":"London","hobby":"gaming"} | -85.809998,113.800003,0.070000,0.924000 |
136+
| 3 | MICHAEL BROWN | male | 23 | 49227.390625 | 23.320000 | 0 | 1697808827 | 677,922 | 3427519,4751707 | {"city":"Tokyo","hobby":"sports"} | -49.570000,17.930000,0.915000,0.134000 |
137+
| 4 | WILLIAM JONES | female | 63 | 92842.390625 | 10.990000 | 1 | 1698885389 | 508,978 | 6898630,9385266 | {"city":"Moscow","hobby":"traveling"} | -53.400002,145.610001,0.963000,0.037000 |
138+
| 5 | DAVID GARCIA | male | 61 | 107581.773438 | 21.840000 | 1 | 1697813613 | 283,309 | 1196507,1440558 | {"city":"Sydney","hobby":"cooking"} | -47.860001,-112.639999,0.934000,0.032000 |
139+
| 6 | RICHARD MILLER | female | 20 | 117996.281250 | 38.330002 | 1 | 1698340704 | 265,889 | 2675141,3102645 | {"city":"New York","hobby":"reading"} | -11.610000,-8.360000,0.096000,0.582000 |
140+
| 7 | CHARLES DAVIS | male | 27 | 107813.171875 | 27.049999 | 1 | 1698120145 | 238,295 | 3072391,8481934 | {"city":"London","hobby":"gaming"} | 49.900002,-142.800003,0.351000,0.224000 |
141+
| 8 | JOSEPH RODRIGUEZ | female | 54 | 47691.359375 | 47.459999 | 0 | 1699347893 | 265,792 | 1604539,7192251 | {"city":"Tokyo","hobby":"sports"} | -32.320000,-117.800003,0.839000,0.001000 |
142+
| 9 | THOMAS MARTINEZ | male | 50 | 47102.101562 | 22.389999 | 0 | 1699290974 | 219,635 | 2380170,7472382 | {"city":"Moscow","hobby":"traveling"} | -2.320000,-63.779999,0.695000,0.625000 |
143+
| 10 | CHRISTOPHER HERNANDEZ | female | 28 | 75799.773438 | 9.530000 | 0 | 1699751633 | 180,966 | 5045063,5375947 | {"city":"Sydney","hobby":"cooking"} | 31.280001,-120.220001,0.578000,0.544000 |
144+
+------+-----------------------+--------+------+---------------+-----------+-----------+------------+---------------+-----------------+---------------------------------------+------------------------------------------+
145+
––– input –––
146+
mysql -P1306 -h0 -e "FLUSH RAMCHUNK name"
147+
––– output –––
148+
––– input –––
149+
mysql -P1306 -h0 -e "FREEZE replication:name"
150+
––– output –––
151+
+-----------------------------------------+-----------------------------------------+
152+
| file | normalized |
153+
+-----------------------------------------+-----------------------------------------+
154+
| /var/log/manticore-1/name/name.meta | /var/log/manticore-1/name/name.meta |
155+
| /var/log/manticore-1/name/name.ram | /var/log/manticore-1/name/name.ram |
156+
| /var/log/manticore-1/name/name.settings | /var/log/manticore-1/name/name.settings |
157+
+-----------------------------------------+-----------------------------------------+
158+
––– input –––
159+
rm -f /tmp/update.sql; for n in `seq 1 100`; do echo "UPDATE name SET gender='updated' WHERE id=$n;" >> /tmp/update.sql; done; mysql -P1306 -h0 -e "source /tmp/update.sql"
160+
––– output –––
161+
––– input –––
162+
sleep 10; mysql -P1306 -h0 -e "SELECT * FROM name ORDER BY id ASC LIMIT 10;"
57163
––– output –––
58-
0
59164
––– input –––
60-
mysql -P9315 -h0 -e "DESCRIBE t"
165+
sleep 1; if timeout 5 mysql -P1306 -h0 -e "SELECT * FROM name ORDER BY id ASC LIMIT 1"; then echo 'Query executed (issue fixed)'; else echo 'Query blocked (issue not fixed)'; fi
61166
––– output –––
62-
+-------+--------+------------+
63-
| Field | Type | Properties |
64-
+-------+--------+------------+
65-
| id | bigint | |
66-
| s | string | |
67-
+-------+--------+------------+
68167
––– input –––
69-
mysql -P9315 -h0 -e "UPDATE t SET s='с' WHERE id=1; FLUSH RAMCHUNK t"; echo $?
168+
sleep 1; kill $UPDATE_PID; wait $UPDATE_PID 2>/dev/null
70169
––– output –––
71-
0

‎test/clt-tests/scripts/load_names_attr.php

Lines changed: 65 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,22 @@
77
'concurrency' => 4,
88
'docs' => 1000000,
99
'start-id' => 1,
10-
'drop-table' => true // Drop and create table by default
10+
'port' => 9306,
11+
'drop-table' => true
1112
];
1213

1314
// Parse command-line arguments
1415
$options = [
15-
'batch-size::', // Optional argument, controlled by --batch-size=
16-
'concurrency::', // Optional argument, controlled by --concurrency=
17-
'docs::', // Optional argument, controlled by --docs=
18-
'min-infix-len::', // Optional argument, controlled by --min-infix-len=
19-
'start-id::', // Optional argument, controlled by --start-id=
20-
'rt-mem-limit::', // Optional argument, controlled by --rt-mem-limit=
21-
'drop-table', // Flag argument, controlled by --drop-table (default true)
22-
'no-drop-table', // Flag to disable dropping and creating the table
23-
'help' // Help argument
16+
'batch-size::',
17+
'concurrency::',
18+
'docs::',
19+
'min-infix-len::',
20+
'start-id::',
21+
'rt-mem-limit::',
22+
'port::',
23+
'drop-table',
24+
'no-drop-table',
25+
'help'
2426
];
2527
$args = getopt("", $options);
2628

@@ -33,7 +35,8 @@
3335
echo " --docs=<number> Total number of documents to insert (default: {$defaults['docs']})\n";
3436
echo " --min-infix-len=<number> Optional minimum infix length for table (default: none, minimum: 2)\n";
3537
echo " --start-id=<number> Starting ID for document insertion (default: {$defaults['start-id']})\n";
36-
echo " --rt-mem-limit=<value> Set the real-time memory limit (optional, default: none)\n";
38+
echo " --rt-mem-limit=<value> Set the real-time memory limit (e.g., 256M, default: none)\n";
39+
echo " --port=<number> MySQL port to connect to (default: {$defaults['port']})\n";
3740
echo " --drop-table Drop and create the table before inserting data (default: true)\n";
3841
echo " --no-drop-table Prevent the table from being dropped and created\n";
3942
echo " --help Show this help message\n";
@@ -54,24 +57,23 @@
5457
$minInfixLen = "";
5558
}
5659

57-
// Set rt-mem-limit argument if specified
58-
$rtMemLimit = isset($args['rt-mem-limit']) ? "rt_mem_limit=" . $args['rt-mem-limit'] : "";
60+
// Set rt-mem-limit argument if specified, with quotes around the value
61+
$rtMemLimit = isset($args['rt-mem-limit']) ? "rt_mem_limit='" . $args['rt-mem-limit'] . "'" : "";
5962

6063
// Determine drop-table behavior
61-
$dropTable = !isset($args['no-drop-table']); // Drop and create table unless --no-drop-table is specified
64+
$dropTable = !isset($args['no-drop-table']);
6265
$batchSize = intval($args['batch-size']);
6366
$concurrency = intval($args['concurrency']);
6467
$docs = intval($args['docs']);
6568
$startId = intval($args['start-id']);
69+
$port = intval($args['port']);
6670

6771
// Connect to database
68-
function connectDb() {
72+
function connectDb($port) {
6973
$host = '127.0.0.1';
70-
$port = '9306';
71-
7274
$m = @mysqli_connect($host, '', '', '', $port);
7375
if (mysqli_connect_error()) {
74-
die("Cannot connect to Manticore\n");
76+
die("Cannot connect to Manticore on port $port: " . mysqli_connect_error() . "\n");
7577
}
7678
return $m;
7779
}
@@ -89,38 +91,75 @@ function process($link, $query) {
8991
// Create connections
9092
$all_links = [];
9193
for ($i = 0; $i < $concurrency; $i++) {
92-
$all_links[] = connectDb();
94+
$all_links[] = connectDb($port);
9395
}
9496

9597
// Initialize table if drop-table is true
9698
$pdo = $all_links[0];
9799
if ($dropTable) {
98100
mysqli_query($pdo, "DROP TABLE IF EXISTS name");
99101
echo "Table 'name' dropped and recreated.\n";
100-
$createTableQuery = "CREATE TABLE name(id INTEGER, username TEXT, s STRING) $minInfixLen expand_keywords='1'";
102+
$createTableQuery = "CREATE TABLE name (
103+
id BIGINT,
104+
username TEXT,
105+
gender STRING,
106+
age INTEGER,
107+
salary FLOAT,
108+
discount FLOAT,
109+
is_active BOOL,
110+
last_login TIMESTAMP,
111+
product_codes MULTI,
112+
large_values MULTI64,
113+
additional_info JSON,
114+
location_vector FLOAT_VECTOR KNN_TYPE='hnsw' KNN_DIMS='4' HNSW_SIMILARITY='l2'
115+
) $minInfixLen expand_keywords='1'";
101116
if ($rtMemLimit) {
102117
$createTableQuery .= " $rtMemLimit";
103118
}
104-
mysqli_query($pdo, $createTableQuery);
119+
mysqli_query($pdo, $createTableQuery) or die("Create table failed: " . mysqli_error($pdo));
105120
}
106121

107122
$batch = [];
108-
$query_start = "INSERT INTO name(id, username, s) VALUES ";
123+
$query_start = "INSERT INTO name(id, username, gender, age, salary, discount, is_active, last_login, product_codes, large_values, additional_info, location_vector) VALUES ";
109124
$names = file('./test/clt-tests/scripts/names.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
110125
$namesCount = count($names);
111126
$surnames = file('./test/clt-tests/scripts/surnames.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
112127
$surnamesCount = count($surnames);
113128

114-
// Ensure repeatable random data
115-
srand(42);
116-
mt_srand(42);
129+
// Sample data for generation
130+
$genders = ['male', 'female'];
131+
$cities = ['New York', 'London', 'Tokyo', 'Moscow', 'Sydney'];
132+
$hobbies = ['reading', 'gaming', 'sports', 'traveling', 'cooking'];
117133

134+
// Ensure repeatable random data
135+
srand(42); // Required for other tasks
136+
mt_srand(42); // Fixed seed for mt_rand
118137
echo "preparing...\n";
119138
$c = $startId; // Start at the specified start ID
120139

121140
$batches = [];
122141
while ($c < $startId + $docs) {
123-
$batch[] = "($c,'" . $names[mt_rand(0, $namesCount - 1)] . ' ' . $surnames[mt_rand(0, $surnamesCount - 1)] . "', 'a')";
142+
$username = $names[($c - 1) % $namesCount] . ' ' . $surnames[($c - 1) % $surnamesCount];
143+
$gender = $genders[($c - 1) % count($genders)];
144+
$age = mt_rand(18, 65);
145+
$salary = mt_rand(30000, 120000) + mt_rand(0, 99) / 100.0;
146+
$discount = mt_rand(0, 50) + mt_rand(0, 99) / 100.0;
147+
$is_active = mt_rand(0, 1);
148+
$last_login = 1700000000 - mt_rand(0, 30 * 24 * 60 * 60); // Fixed base timestamp
149+
$product_codes = '(' . mt_rand(100, 999) . ',' . mt_rand(100, 999) . ')';
150+
$large_values = '(' . mt_rand(1000000, 9999999) . ',' . mt_rand(1000000, 9999999) . ')';
151+
$additional_info = json_encode([
152+
'city' => $cities[($c - 1) % count($cities)],
153+
'hobby' => $hobbies[($c - 1) % count($hobbies)]
154+
]);
155+
$location_vector = '(' . implode(',', array(
156+
mt_rand(-90, 90) + mt_rand(0, 99) / 100.0,
157+
mt_rand(-180, 180) + mt_rand(0, 99) / 100.0,
158+
mt_rand(0, 1000) / 1000.0,
159+
mt_rand(0, 1000) / 1000.0
160+
)) . ')';
161+
162+
$batch[] = "($c,'$username','$gender',$age,$salary,$discount,$is_active,$last_login,$product_codes,$large_values,'$additional_info',$location_vector)";
124163
$c++;
125164
if (($c - $startId) % 1000 == 0) echo "\r" . (($c - $startId) / $docs * 100) . "% ";
126165
if (count($batch) == $batchSize) {

0 commit comments

Comments
 (0)