Skip to content

Commit a9096f6

Browse files
k1LoWk1LoW
authored
Merge pull request #609 from k1LoW/clickhouse-sample
Add ClickHouse sample
2 parents 1180eee + 38360df commit a9096f6

20 files changed

+642
-2
lines changed

‎Makefile

+2
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ doc: build doc_sqlite
5959
$(TBLS) doc ms://SA:MSSQLServer-Passw0rd@localhost:11433/testdb -c testdata/test_tbls_mssql.yml -f sample/mssql
6060
$(TBLS) doc mongodb://mongoadmin:secret@localhost:27017/test?authSource=admin -f sample/mongo
6161
env AWS_ENDPOINT_URL=http://localhost:18000 $(TBLS) doc dynamodb://ap-northeast-1 -c testdata/test_tbls_dynamodb.yml -f sample/dynamodb
62+
$(TBLS) doc clickhouse://default@localhost:9000/testdb -f sample/clickhouse
6263
$(TBLS) doc pg://postgres:pgpass@localhost:55413/testdb?sslmode=disable -c testdata/test_tbls_postgres.yml -j -f sample/adjust
6364
$(TBLS) doc my://root:mypass@localhost:33308/testdb -c testdata/test_tbls.yml -t png -f sample/png
6465
$(TBLS) doc my://root:mypass@localhost:33308/testdb -c testdata/test_tbls.yml -t mermaid -f sample/mermaid
@@ -84,6 +85,7 @@ testdoc: build testdoc_sqlite
8485
$(TBLS) diff ms://SA:MSSQLServer-Passw0rd@localhost:11433/testdb -c testdata/test_tbls_mssql.yml sample/mssql
8586
$(TBLS) diff mongodb://mongoadmin:secret@localhost:27017/test?authSource=admin sample/mongo
8687
env AWS_ENDPOINT_URL=http://localhost:18000 $(TBLS) diff dynamodb://ap-northeast-1 -c testdata/test_tbls_dynamodb.yml sample/dynamodb
88+
$(TBLS) diff clickhouse://default@localhost:9000/testdb sample/clickhouse
8789
$(TBLS) diff pg://postgres:pgpass@localhost:55413/testdb?sslmode=disable -c testdata/test_tbls_postgres.yml -j sample/adjust
8890
$(TBLS) diff my://root:mypass@localhost:33308/testdb -c testdata/test_tbls.yml -t png sample/png
8991
$(TBLS) diff my://root:mypass@localhost:33308/testdb -c testdata/exclude_test_tbls.yml sample/exclude

‎docker-compose.yml ‎compose.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
version: '3.4'
2-
31
services:
42
postgres95:
53
image: postgres:9.5
4+
platform: linux/amd64
65
restart: always
76
ports:
87
- "55432:5432"
@@ -21,6 +20,7 @@ services:
2120
- POSTGRES_DB=testdb
2221
mysql56:
2322
image: mysql:5.6
23+
platform: linux/amd64
2424
restart: always
2525
ports:
2626
- "33306:3306"

‎drivers/clickhouse/clickhouse.go

+19
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,17 @@ package clickhouse
33
import (
44
"database/sql"
55
"fmt"
6+
"regexp"
7+
68
_ "github.com/ClickHouse/clickhouse-go/v2"
79
"github.com/k1LoW/tbls/dict"
810
"github.com/k1LoW/tbls/schema"
911
"github.com/pkg/errors"
12+
"github.com/samber/lo"
1013
)
1114

15+
var shadowTableRe = regexp.MustCompile(`^\.inner_id\.`)
16+
1217
// ClickHouse struct
1318
type ClickHouse struct {
1419
db *sql.DB
@@ -35,6 +40,7 @@ func (ch *ClickHouse) Analyze(s *schema.Schema) error {
3540
tableSortingKeys := make(map[string]*schema.Constraint)
3641
tablePrimaryKeys := make(map[string]*schema.Constraint)
3742
tableSamplingKeys := make(map[string]*schema.Constraint)
43+
var filtered []string
3844

3945
tableRows, err := ch.db.Query(`
4046
SELECT
@@ -83,6 +89,11 @@ WHERE database = ?
8389
Comment: tableComment,
8490
}
8591

92+
if shadowTableRe.MatchString(tableName) {
93+
filtered = append(filtered, tableName)
94+
continue
95+
}
96+
8697
s.Tables = append(s.Tables, table)
8798

8899
if tablePartitionKey != "" {
@@ -191,6 +202,10 @@ ORDER BY table
191202
Nullable: false,
192203
}
193204

205+
if lo.Contains(filtered, colTable) {
206+
continue
207+
}
208+
194209
table, err := s.FindTableByName(colTable)
195210
if err != nil {
196211
return errors.WithStack(err)
@@ -276,6 +291,10 @@ ORDER BY table
276291
return errors.WithStack(err)
277292
}
278293

294+
if lo.Contains(filtered, idxTable) {
295+
continue
296+
}
297+
279298
table, err := s.FindTableByName(idxTable)
280299
if err != nil {
281300
return errors.WithStack(err)

‎sample/clickhouse/README.md

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# testdb
2+
3+
## Tables
4+
5+
| Name | Columns | Comment | Type |
6+
| ---- | ------- | ------- | ---- |
7+
| [id_value_dictionary](id_value_dictionary.md) | 2 | | Dictionary |
8+
| [materialized_view](materialized_view.md) | 2 | | MaterializedView |
9+
| [numbers_table](numbers_table.md) | 1 | | SystemNumbers |
10+
| [source_table](source_table.md) | 2 | | MergeTree |
11+
| [t1](t1.md) | 1 | | Memory |
12+
| [table_name](table_name.md) | 8 | comment for table | MergeTree |
13+
| [view](view.md) | 5 | | View |
14+
15+
## Stored procedures and functions
16+
17+
| Name | ReturnType | Arguments | Type |
18+
| ---- | ------- | ------- | ---- |
19+
| linear_equation | | | |
20+
21+
## Relations
22+
23+
![er](schema.svg)
24+
25+
---
26+
27+
> Generated by [tbls](https://github.com/k1LoW/tbls)

‎sample/clickhouse/id_value_dictionary.md

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# id_value_dictionary
2+
3+
## Description
4+
5+
<details>
6+
<summary><strong>Table Definition</strong></summary>
7+
8+
```sql
9+
CREATE DICTIONARY testdb.id_value_dictionary (`id` UInt64, `value` String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'source_table')) LIFETIME(MIN 0 MAX 1000) LAYOUT(FLAT())
10+
```
11+
12+
</details>
13+
14+
## Columns
15+
16+
| Name | Type | Default | Nullable | Children | Parents | Comment |
17+
| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
18+
| id | UInt64 | | false | | | |
19+
| value | String | | false | | | |
20+
21+
## Relations
22+
23+
![er](id_value_dictionary.svg)
24+
25+
---
26+
27+
> Generated by [tbls](https://github.com/k1LoW/tbls)

‎sample/clickhouse/id_value_dictionary.svg

+29
Loading

‎sample/clickhouse/materialized_view.md

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# materialized_view
2+
3+
## Description
4+
5+
<details>
6+
<summary><strong>Table Definition</strong></summary>
7+
8+
```sql
9+
CREATE MATERIALIZED VIEW testdb.materialized_view (`name1` UInt64, `name2` Nullable(String)) ENGINE = Memory AS SELECT name1, name2 FROM testdb.table_name ORDER BY name1 DESC
10+
```
11+
12+
</details>
13+
14+
## Columns
15+
16+
| Name | Type | Default | Nullable | Children | Parents | Comment |
17+
| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
18+
| name1 | UInt64 | | false | | | |
19+
| name2 | Nullable(String) | | false | | | |
20+
21+
## Referenced Tables
22+
23+
| Name | Columns | Comment | Type |
24+
| ---- | ------- | ------- | ---- |
25+
| [table_name](table_name.md) | 8 | comment for table | MergeTree |
26+
27+
## Relations
28+
29+
![er](materialized_view.svg)
30+
31+
---
32+
33+
> Generated by [tbls](https://github.com/k1LoW/tbls)

‎sample/clickhouse/materialized_view.svg

+29
Loading

‎sample/clickhouse/numbers_table.md

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# numbers_table
2+
3+
## Description
4+
5+
<details>
6+
<summary><strong>Table Definition</strong></summary>
7+
8+
```sql
9+
CREATE TABLE testdb.numbers_table (`number` UInt64) AS numbers(100)
10+
```
11+
12+
</details>
13+
14+
## Columns
15+
16+
| Name | Type | Default | Nullable | Children | Parents | Comment |
17+
| ---- | ---- | ------- | -------- | -------- | ------- | ------- |
18+
| number | UInt64 | | false | | | |
19+
20+
## Relations
21+
22+
![er](numbers_table.svg)
23+
24+
---
25+
26+
> Generated by [tbls](https://github.com/k1LoW/tbls)

‎sample/clickhouse/numbers_table.svg

+26
Loading

‎sample/clickhouse/schema.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"name":"testdb","desc":"","tables":[{"name":"id_value_dictionary","type":"Dictionary","comment":"","columns":[{"name":"id","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"value","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE DICTIONARY testdb.id_value_dictionary (`id` UInt64, `value` String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'source_table')) LIFETIME(MIN 0 MAX 1000) LAYOUT(FLAT())"},{"name":"materialized_view","type":"MaterializedView","comment":"","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"name2","type":"Nullable(String)","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE MATERIALIZED VIEW testdb.materialized_view (`name1` UInt64, `name2` Nullable(String)) ENGINE = Memory AS SELECT name1, name2 FROM testdb.table_name ORDER BY name1 DESC","referenced_tables":["table_name"]},{"name":"numbers_table","type":"SystemNumbers","comment":"","columns":[{"name":"number","type":"UInt64","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE TABLE testdb.numbers_table (`number` UInt64) AS numbers(100)"},{"name":"source_table","type":"MergeTree","comment":"","columns":[{"name":"id","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"value","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[{"name":"sorting key","type":"SORTING KEY","def":"ORDER BY (id)","table":"source_table","referenced_table":null,"columns":["id"],"referenced_columns":null,"comment":""},{"name":"primary key","type":"PRIMARY KEY","def":"PRIMARY KEY (id)","table":"source_table","referenced_table":null,"columns":["id"],"referenced_columns":null,"comment":""}],"triggers":[],"def":"CREATE TABLE testdb.source_table (`id` UInt64, `value` String) ENGINE = MergeTree PRIMARY KEY id ORDER BY id SETTINGS index_granularity = 8192"},{"name":"t1","type":"Memory","comment":"","columns":[{"name":"x","type":"String","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE TABLE testdb.t1 (`x` String) ENGINE = Memory"},{"name":"table_name","type":"MergeTree","comment":"comment for table","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":"comment for column 1"},{"name":"name2","type":"Nullable(String)","nullable":false,"default":"DEFAULT 'column 2'","comment":"comment for column 2"},{"name":"name3","type":"LowCardinality(String)","nullable":false,"default":"MATERIALIZED upper(name2)","comment":"comment for column 3"},{"name":"name4","type":"SimpleAggregateFunction(sum, Float64)","nullable":false,"default":null,"comment":""},{"name":"name5","type":"DateTime","nullable":false,"default":"DEFAULT now()","comment":""},{"name":"name6","type":"String","nullable":false,"default":"ALIAS formatReadableSize(name1)","comment":""},{"name":"name7","type":"String","nullable":false,"default":"MATERIALIZED hex(name1)","comment":""},{"name":"name8","type":"FixedString(4)","nullable":false,"default":"DEFAULT unhex(name7)","comment":""}],"indexes":[{"name":"idx1","def":"bloom_filter(0.01)","table":"table_name","columns":["name1"],"comment":""},{"name":"idx2","def":"minmax","table":"table_name","columns":["name1 * 2"],"comment":""},{"name":"idx3","def":"set(1000)","table":"table_name","columns":["name1 * length(name2)"],"comment":""}],"constraints":[{"name":"partition key","type":"PARTITION KEY","def":"PARTITION BY ((name1, name3, name5))","table":"table_name","referenced_table":null,"columns":["name1","name3","name5"],"referenced_columns":null,"comment":""},{"name":"sorting key","type":"SORTING KEY","def":"ORDER BY (name1, name5)","table":"table_name","referenced_table":null,"columns":["name1","name5"],"referenced_columns":null,"comment":""},{"name":"primary key","type":"PRIMARY KEY","def":"PRIMARY KEY (name1, name5)","table":"table_name","referenced_table":null,"columns":["name1","name5"],"referenced_columns":null,"comment":""},{"name":"sampling key","type":"SAMPLING KEY","def":"SAMPLE BY (name1)","table":"table_name","referenced_table":null,"columns":["name1"],"referenced_columns":null,"comment":""}],"triggers":[],"def":"CREATE TABLE testdb.table_name (`name1` UInt64 COMMENT 'comment for column 1', `name2` Nullable(String) DEFAULT 'column 2' COMMENT 'comment for column 2' CODEC(ZSTD(1)), `name3` LowCardinality(String) MATERIALIZED upper(name2) COMMENT 'comment for column 3', `name4` SimpleAggregateFunction(sum, Float64) TTL name5 + toIntervalDay(1), `name5` DateTime DEFAULT now(), `name6` String ALIAS formatReadableSize(name1), `name7` String MATERIALIZED hex(name1), `name8` FixedString(4) DEFAULT unhex(name7), INDEX idx1 name1 TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx2 name1 * 2 TYPE minmax GRANULARITY 3, INDEX idx3 name1 * length(name2) TYPE set(1000) GRANULARITY 4, PROJECTION projection_name_1 (SELECT name1, name2, name3 ORDER BY name1)) ENGINE = MergeTree PARTITION BY (name1, name3, name5) PRIMARY KEY (name1, name5) ORDER BY (name1, name5) SAMPLE BY name1 SETTINGS index_granularity = 8192 COMMENT 'comment for table'"},{"name":"view","type":"View","comment":"","columns":[{"name":"name1","type":"UInt64","nullable":false,"default":null,"comment":""},{"name":"name2","type":"Nullable(String)","nullable":false,"default":null,"comment":""},{"name":"name4","type":"SimpleAggregateFunction(sum, Float64)","nullable":false,"default":null,"comment":""},{"name":"name5","type":"DateTime","nullable":false,"default":null,"comment":""},{"name":"name8","type":"FixedString(4)","nullable":false,"default":null,"comment":""}],"indexes":[],"constraints":[],"triggers":[],"def":"CREATE VIEW testdb.view (`name1` UInt64, `name2` Nullable(String), `name4` SimpleAggregateFunction(sum, Float64), `name5` DateTime, `name8` FixedString(4)) AS SELECT * FROM testdb.table_name"}],"relations":[],"functions":[{"name":"linear_equation","return_type":"","arguments":"","type":""}],"driver":{"name":"clickhouse","database_version":"24.4.4.85","meta":{"dict":{"Functions":"Stored procedures and functions"}}}}

0 commit comments

Comments
 (0)