Test/sharding syntax

.github/workflows/clt_nightly.yml

@@ -32,7 +32,7 @@ jobs:
         image: [ "centos:7", "almalinux:8", "almalinux:9", "oraclelinux:9", "amazonlinux:latest" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/clt@0.1.13
+      - uses: manticoresoftware/clt@0.1.15
         with:
           ref: manticore-6.2.12
           image: ${{ matrix.image }}
@@ -45,7 +45,7 @@ jobs:
         image: [ "ubuntu:bionic", "ubuntu:focal", "ubuntu:jammy", "debian:buster", "debian:bullseye", "debian:bookworm" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/clt@0.1.13
+      - uses: manticoresoftware/clt@0.1.15
         with:
           ref: manticore-6.2.12
           image: ${{ matrix.image }}

.github/workflows/pack_publish.yml

@@ -297,7 +297,7 @@ jobs:
         image: [ "centos:7", "almalinux:8", "almalinux:9", "oraclelinux:9", "amazonlinux:latest" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/clt@0.1.11
+      - uses: manticoresoftware/clt@0.1.5
         with:
           image: ${{ matrix.image }}
           test_prefix: test/clt-tests/installation/rhel-dev-
@@ -312,7 +312,7 @@ jobs:
         image: [ "ubuntu:bionic", "ubuntu:focal", "ubuntu:jammy", "debian:buster", "debian:bullseye", "debian:bookworm" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/clt@0.1.11
+      - uses: manticoresoftware/clt@0.1.5
         with:
           image: ${{ matrix.image }}
           test_prefix: test/clt-tests/installation/deb-dev-

.github/workflows/test.yml

@@ -23,18 +23,6 @@ concurrency:
 
 jobs:
 
-  commit_info:
-    name: Commit info
-    runs-on: ubuntu-22.04
-    steps:
-      - run: |
-          echo '# Automated Tests of commit ${{ github.sha }}' >> $GITHUB_STEP_SUMMARY
-          echo '${{ github.event.head_commit.message }}' >> $GITHUB_STEP_SUMMARY
-          echo '* Commit URL: [${{ github.sha }}](/${{ github.repository }}/commit/${{ github.sha }})' >> $GITHUB_STEP_SUMMARY
-          echo '* Initiated by: [@${{ github.actor }}](https://github.com/${{ github.actor }})' >> $GITHUB_STEP_SUMMARY
-          echo '* Ref: ${{ github.ref_type }} "${{ github.ref_name }}"' >> $GITHUB_STEP_SUMMARY
-          echo '* Attempt: ${{ github.run_attempt }}' >> $GITHUB_STEP_SUMMARY
-
   win_bundle:
     name: Windows supplementary files preparation
     if: |
@@ -422,7 +410,7 @@ jobs:
 
 
   clt_core_test:
-    name: CLT test of core logic of Manticore Search
+    name: CLT test core logic
     needs: build_test_kit_docker
     runs-on: ubuntu-22.04
     steps:
@@ -431,13 +419,13 @@ jobs:
           artifact: manticore_test_kit.img
           image: test-kit:img
           test_prefix: test/clt-tests/core/
-
+          
   clt_logstash_test:
     name: CLT test on Logstash and Filebeat
     needs: build_test_kit_docker
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/clt@0.1.13
+      - uses: manticoresoftware/clt@0.1.15
         with:
           artifact: manticore_test_kit.img
           image: test-kit:img
@@ -459,7 +447,7 @@ jobs:
     needs: build_test_kit_docker
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/clt@0.1.13
+      - uses: manticoresoftware/clt@0.1.5
         with:
           artifact: manticore_test_kit.img
           image: test-kit:img
@@ -470,12 +458,23 @@ jobs:
     needs: build_test_kit_docker
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/clt@0.1.13
+      - uses: manticoresoftware/clt@0.1.15
         with:
           artifact: manticore_test_kit.img
           image: test-kit:img
           test_prefix: test/clt-tests/expected-errors/
 
+  clt_sharding_syntax_test:
+    name: CLT test syntax of autosharding
+    needs: build_test_kit_docker
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: manticoresoftware/clt@0.1.15
+        with:
+          artifact: manticore_test_kit.img
+          image: test-kit:img
+          test_prefix: test/clt-tests/sharding/syntax/
+
   build_aarch64:
     name: Linux aarch64 build
     if: |

.gitignore

@@ -76,5 +76,6 @@
 
 /local_*_src.txt
 /test/clt-tests/**/*.rep
+/test/clt-tests/**/*.cmp
 /clt
-
+**/.DS_Store

test/clt-tests/sharding/base/config/searchd-with-flexible-ports.conf

@@ -0,0 +1,20 @@
+#!/bin/sh
+cat << EOF
+common {
+	plugin_dir = /usr/local/lib/manticore
+	lemmatizer_base = /usr/share/manticore/morph/
+}
+searchd {
+	listen = ${INSTANCE}306:mysql41
+	listen = ${INSTANCE}312
+	listen = ${INSTANCE}308:http
+	log = /var/log/manticore-${INSTANCE}/searchd.log
+	query_log = /var/log/manticore-${INSTANCE}/query.log
+	pid_file = /var/log/manticore-${INSTANCE}/searchd.pid
+	data_dir = /var/log/manticore-${INSTANCE}
+	query_log_format = sphinxql
+	query_log_commands = 1
+	buddy_path = manticore-executor /workdir/src/main.php
+}
+
+EOF

test/clt-tests/sharding/base/flexible/create-cluster.recb

@@ -0,0 +1,9 @@
+––– input –––
+mysql -h0 -P1306 -e "create cluster ${CLUSTER_NAME}"
+––– output –––
+––– input –––
+mysql -h0 -P1306 -e "show status like 'cluster_${CLUSTER_NAME}_status'\G"
+––– output –––
+*************************** 1. row ***************************
+Counter: cluster_#!/[a-z]+/!#_status
+Value: primary

test/clt-tests/sharding/base/flexible/join-cluster-on-all-nodes.recb

@@ -0,0 +1,9 @@
+––– input –––
+for n in `seq 2 $INSTANCE`; do mysql -h0 -P${n}306 -e "join cluster ${CLUSTER_NAME} at '127.0.0.1:1312'"; done;
+––– output –––
+––– input –––
+mysql -h0 -P${INSTANCE}306 -e "show status like 'cluster_${CLUSTER_NAME}_status'\G"
+––– output –––
+*************************** 1. row ***************************
+Counter: cluster_#!/[a-z]+/!#_status
+Value: primary

test/clt-tests/sharding/base/flexible/start-searchd-timeout.recb

@@ -0,0 +1,19 @@
+––– input –––
+mkdir -p /var/{run,lib,log}/manticore-${INSTANCE}/{a,b,c,d,e,f,g,h,i,j}
+––– output –––
+––– input –––
+searchd -c test/clt-tests/sharding/base/config/searchd-with-flexible-ports.conf
+––– output –––
+Manticore %{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE}) (knn %{SEMVER} %{COMMITDATE})
+Copyright (c) 2001-2016, Andrew Aksyonoff
+Copyright (c) 2008-2016, Sphinx Technologies Inc (http://sphinxsearch.com)
+Copyright (c) 2017-%{YEAR}, Manticore Software LTD (https://manticoresearch.com)
+[#!/[0-9]{2}:[0-9]{2}.[0-9]{3}/!#] [%{NUMBER}] using config file '%{PATH}' (%{NUMBER} chars)...
+starting daemon version '%{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE}) (knn %{SEMVER} %{COMMITDATE})' ...
+listening on all interfaces for mysql, port=%{NUMBER}
+listening on all interfaces for sphinx and http(s), port=%{NUMBER}
+listening on all interfaces for sphinx and http(s), port=%{NUMBER}
+––– input –––
+timeout 10 tail -n 100 -f /var/log/manticore-${INSTANCE}/searchd.log | grep -qm1 '\[BUDDY\] started' && echo 'Buddy started!' || echo 'Timeout or failed!'
+––– output –––
+Buddy started!

test/clt-tests/sharding/base/flexible/start-searchd.recb

@@ -0,0 +1,19 @@
+––– input –––
+mkdir -p /var/{run,lib,log}/manticore-${INSTANCE}/{a,b,c,d,e,f,g,h,i,j}
+––– output –––
+––– input –––
+stdbuf -oL searchd -c test/clt-tests/sharding/base/config/searchd-with-flexible-ports.conf | grep -v precach
+––– output –––
+Manticore %{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE}) (knn %{SEMVER} %{COMMITDATE})
+Copyright (c) 2001-2016, Andrew Aksyonoff
+Copyright (c) 2008-2016, Sphinx Technologies Inc (http://sphinxsearch.com)
+Copyright (c) 2017-%{YEAR}, Manticore Software LTD (https://manticoresearch.com)
+[#!/[A-Za-z]{3}\s[A-Za-z]{3}\s[0-9]{2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}\s[0-9]{4}/!#] [%{NUMBER}] using config file '%{PATH}' (%{NUMBER} chars)...
+starting daemon version '%{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE}) (knn %{SEMVER} %{COMMITDATE})' ...
+listening on all interfaces for mysql, port=%{NUMBER}
+listening on all interfaces for sphinx and http(s), port=%{NUMBER}
+listening on all interfaces for sphinx and http(s), port=%{NUMBER}
+––– input –––
+tail -n 100 -f /var/log/manticore-${INSTANCE}/searchd.log | grep -qm1 'started v' && echo "Buddy started!"
+––– output –––
+Buddy started!

test/clt-tests/sharding/base/flexible/stop-searchd.recb

@@ -0,0 +1,9 @@
+––– input –––
+searchd --stop -c test/clt-tests/sharding/base/config/searchd-with-flexible-ports.conf
+––– output –––
+Manticore %{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE}) (knn %{SEMVER} %{COMMITDATE})
+Copyright (c) 2001-2016, Andrew Aksyonoff
+Copyright (c) 2008-2016, Sphinx Technologies Inc (http://sphinxsearch.com)
+Copyright (c) 2017-%{YEAR}, Manticore Software LTD (https://manticoresearch.com)
+[#!/[0-9]{2}:[0-9]{2}.[0-9]{3}/!#] [%{NUMBER}] using config file '%{PATH}' (%{NUMBER} chars)...
+[#!/[0-9]{2}:[0-9]{2}.[0-9]{3}/!#] [%{NUMBER}] stop: successfully sent SIGTERM to pid %{NUMBER}

test/clt-tests/sharding/base/init.recb

@@ -0,0 +1,27 @@
+––– input –––
+set -b
+––– output –––
+––– input –––
+apt-get update -y > /dev/null; echo $?
+––– output –––
+0
+––– input –––
+apt-get install -y git > /dev/null; echo $?
+––– output –––
+0
+––– input –––
+git clone -b fix/sharding -q https://github.com/manticoresoftware/manticoresearch-buddy.git /workdir; echo $?
+––– output –––
+0
+––– input –––
+cd /workdir; echo $?
+––– output –––
+0
+––– input –––
+COMPOSER_HOME="$HOME/.config/composer" composer install -q; echo $?
+––– output –––
+0
+––– input –––
+cd /.clt; echo $?
+––– output –––
+0

test/clt-tests/sharding/syntax/queries-negative-test.rec

@@ -0,0 +1,45 @@
+––– block: ../base/init –––
+––– input –––
+export INSTANCE=1
+––– output –––
+––– block: ../base/flexible/start-searchd –––
+––– input –––
+export INSTANCE=2
+––– output –––
+––– block: ../base/flexible/start-searchd –––
+––– input –––
+export INSTANCE=3
+––– output –––
+––– block: ../base/flexible/start-searchd –––
+––– input –––
+mysql -h0 -P1306 -e "create table h random=5"
+––– output –––
+ERROR 1064 (42000) at line 1: You cannot set rf greater than 1 when creating single node sharded table.
+––– input –––
+mysql -h0 -P1306 -e "create table t(id bigint) shards=a rf=b"
+––– output –––
+ERROR 1064 (42000) at line 1: Failed to parse query
+––– input –––
+mysql -h0 -P1306 -e "create table u(id bigint) rf=1 shards=10"
+––– output –––
+––– input –––
+mysql -h0 -P1306 -e "create table c:m SHARDS=10 RF=2"
+––– output –––
+ERROR 1064 (42000) at line 1: Cluster 'c' does not exist
+––– input –––
+mysql -h0 -P1306 -e "create table l(id bigint) SHARDS=-10 RF=-1"
+––– output –––
+ERROR 1064 (42000) at line 1: Failed to parse query
+––– input –––
+mysql -h0 -P1306 -e "create table n(id bigint) shards=10 rf=2"
+––– output –––
+ERROR 1064 (42000) at line 1: You cannot set rf greater than 1 when creating single node sharded table.
+––– input –––
+––– input –––
+export CLUSTER_NAME=c
+––– output –––
+––– block: ../base/flexible/create-cluster –––
+––– input –––
+mysql -h0 -P1306 -e "create table c:m(id bigint) SHARDS=10 RF=2"
+––– output –––
+ERROR 1064 (42000) at line 1: Waiting timeout exceeded.

test/clt-tests/sharding/syntax/queries-syntax-quotes.rec

@@ -0,0 +1,106 @@
+––– block: ../base/init –––
+––– input –––
+export INSTANCE=1
+––– output –––
+––– block: ../base/flexible/start-searchd –––
+––– input –––
+mysql -h0 -P1306 -e "create table u(id bigint) shards=10 rf=1"; echo $?
+––– output –––
+0
+––– input –––
+mysql -h0 -P1306 -e "SHOW TABLES\G"
+––– output –––
+ *************************** 1. row ***************************
+Index: sharding_queue
+Type: rt
+*************************** 2. row ***************************
+Index: sharding_state
+Type: rt
+*************************** 3. row ***************************
+Index: sharding_table
+Type: rt
+*************************** 4. row ***************************
+Index: u
+Type: distributed
+*************************** 5. row ***************************
+Index: u_s0
+Type: rt
+*************************** 6. row ***************************
+Index: u_s1
+Type: rt
+*************************** 7. row ***************************
+Index: u_s2
+Type: rt
+*************************** 8. row ***************************
+Index: u_s3
+Type: rt
+*************************** 9. row ***************************
+Index: u_s4
+Type: rt
+*************************** 10. row ***************************
+Index: u_s5
+Type: rt
+*************************** 11. row ***************************
+Index: u_s6
+Type: rt
+*************************** 12. row ***************************
+Index: u_s7
+Type: rt
+*************************** 13. row ***************************
+Index: u_s8
+Type: rt
+*************************** 14. row ***************************
+Index: u_s9
+Type: rt
+––– input –––
+mysql -h0 -P1306 -e "create table b(id bigint) shards='10' rf='1'"; echo $?
+––– output –––
+0
+––– input –––
+mysql -h0 -P1306 -e "SHOW TABLES\G"
+––– output –––
+*************************** 1. row ***************************
+Index: b
+Type: rt
+*************************** 2. row ***************************
+Index: sharding_queue
+Type: rt
+*************************** 3. row ***************************
+Index: sharding_state
+Type: rt
+*************************** 4. row ***************************
+Index: sharding_table
+Type: rt
+*************************** 5. row ***************************
+Index: u
+Type: distributed
+*************************** 6. row ***************************
+Index: u_s0
+Type: rt
+*************************** 7. row ***************************
+Index: u_s1
+Type: rt
+*************************** 8. row ***************************
+Index: u_s2
+Type: rt
+*************************** 9. row ***************************
+Index: u_s3
+Type: rt
+*************************** 10. row ***************************
+Index: u_s4
+Type: rt
+*************************** 11. row ***************************
+Index: u_s5
+Type: rt
+*************************** 12. row ***************************
+Index: u_s6
+Type: rt
+*************************** 13. row ***************************
+Index: u_s7
+Type: rt
+*************************** 14. row ***************************
+Index: u_s8
+Type: rt
+*************************** 15. row ***************************
+Index: u_s9
+Type: rt