Skip to content

Commit a188494

Browse files
fanyang01fanyang01
authored
test: add workflow to validate HTAP guides (#243)
* test: add HTAP workflow * migrate to docker compose v2
1 parent 3ad3ba0 commit a188494

File tree

8 files changed

+196
-28
lines changed

8 files changed

+196
-28
lines changed

‎.github/workflows/mysql-htap-maxscale-setup.yml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: MySQL HTAP MaxScale Test
2+
3+
on:
4+
push:
5+
branches: [ "main" ]
6+
7+
jobs:
8+
htap-test:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- uses: actions/checkout@v4
12+
13+
# Install dependencies
14+
- name: Install dependencies
15+
run: |
16+
sudo apt-get update
17+
sudo apt-get install -y mysql-client
18+
curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip
19+
unzip duckdb_cli-linux-amd64.zip
20+
chmod +x duckdb
21+
sudo mv duckdb /usr/local/bin
22+
23+
- name: Launch HTAP cluster
24+
run: |
25+
cd devtools/htap-setup-mysql/maxscale
26+
docker compose up -d --wait
27+
28+
- name: Verify HTAP setup
29+
run: |
30+
# Save SELECT count before executing the query
31+
docker exec maxscale maxctrl api get servers/myduck-server data.attributes.statistics.total_connections | tee select_count_before.txt
32+
33+
# Execute READ statement
34+
mysql -h127.0.0.1 -P14000 -ulol -plol -e "SELECT * FROM db01.test;"
35+
36+
# Save SELECT count after executing the query
37+
docker exec maxscale maxctrl api get servers/myduck-server data.attributes.statistics.total_connections | tee select_count_after.txt
38+
39+
# Read counts from files
40+
before_count=$(cat select_count_before.txt)
41+
after_count=$(cat select_count_after.txt)
42+
43+
# Calculate the difference
44+
diff=$((after_count - before_count))
45+
46+
# Verify that SELECT count increased by 1
47+
if [ "$diff" -eq 1 ]; then
48+
echo 'HTAP setup verification successful.'
49+
else
50+
echo 'HTAP setup verification failed.'
51+
exit 1
52+
fi
53+
54+
- name: Cleanup
55+
if: always()
56+
run: |
57+
cd devtools/htap-setup-mysql/maxscale
58+
docker compose down

‎.github/workflows/mysql-htap-proxysql-setup.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: MySQL HTAP ProxySQL Test
2+
3+
on:
4+
push:
5+
branches: [ "main" ]
6+
7+
jobs:
8+
htap-test:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- uses: actions/checkout@v4
12+
13+
# Install dependencies
14+
- name: Install dependencies
15+
run: |
16+
sudo apt-get update
17+
sudo apt-get install -y mysql-client
18+
curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip
19+
unzip duckdb_cli-linux-amd64.zip
20+
chmod +x duckdb
21+
sudo mv duckdb /usr/local/bin
22+
23+
- name: Launch HTAP cluster
24+
run: |
25+
cd devtools/htap-setup-mysql/proxysql
26+
docker compose up -d --wait
27+
sleep 10
28+
29+
- name: Verify HTAP setup
30+
run: |
31+
# Save stats before SELECT
32+
mysql -h127.0.0.1 -P16032 -uradmin -pradmin --batch --raw -e "SELECT srv_host, srv_port, Queries FROM stats_mysql_connection_pool;" | tee stats_before.csv
33+
34+
# Execute READ statement
35+
mysql -h127.0.0.1 -P16033 -ulol -plol -e "SELECT * FROM db01.test;"
36+
37+
# Save stats after SELECT
38+
mysql -h127.0.0.1 -P16032 -uradmin -pradmin --batch --raw -e "SELECT srv_host, srv_port, Queries FROM stats_mysql_connection_pool;" | tee stats_after.csv
39+
40+
# Use DuckDB to check if Queries count increased for the read server
41+
duckdb --csv -c "
42+
CREATE TABLE before AS FROM 'stats_before.csv';
43+
CREATE TABLE after AS FROM 'stats_after.csv';
44+
SELECT (after.Queries - before.Queries) AS diff
45+
FROM before JOIN after USING(srv_host, srv_port)
46+
WHERE srv_host = 'myduck';
47+
" | tail -n 1 | tee query_cnt_diff.txt
48+
49+
# Verify that #queries increased by 1
50+
if grep -q '^1$' query_cnt_diff.txt; then
51+
echo 'HTAP setup verification successful.'
52+
else
53+
echo 'HTAP setup verification failed.'
54+
exit 1
55+
fi
56+
57+
- name: Cleanup
58+
if: always()
59+
run: |
60+
cd devtools/htap-setup-mysql/proxysql
61+
docker compose down

‎.github/workflows/pg-htap-pgpool-setup.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: PG HTAP PGPool-II Test
2+
3+
on:
4+
push:
5+
branches: [ "main" ]
6+
7+
jobs:
8+
htap-test:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- uses: actions/checkout@v4
12+
13+
- name: Set up Docker
14+
uses: docker/setup-buildx-action@v3
15+
16+
- name: Install dependencies
17+
run: |
18+
# Install DuckDB
19+
curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip
20+
unzip duckdb_cli-linux-amd64.zip
21+
chmod +x duckdb
22+
sudo mv duckdb /usr/local/bin
23+
24+
- name: Launch HTAP cluster
25+
run: |
26+
cd devtools/htap-setup-pg/pgpool2
27+
docker compose up -d --wait
28+
29+
- name: Verify HTAP setup
30+
run: |
31+
# Save SHOW POOL_NODES output before SELECT
32+
docker exec htap-pgpool bash -c "PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c 'SHOW POOL_NODES;' -F ',' --no-align --field-separator ',' --pset footer=off" | tee pool_nodes_before.csv
33+
34+
# Execute READ statement
35+
docker exec htap-pgpool bash -c "PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c 'SELECT * FROM test;'"
36+
37+
# Save SHOW POOL_NODES output after SELECT
38+
docker exec htap-pgpool bash -c "PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c 'SHOW POOL_NODES;' -F ',' --no-align --field-separator ',' --pset footer=off" | tee pool_nodes_after.csv
39+
40+
# Use DuckDB to check if select_cnt increased
41+
duckdb --csv -c "
42+
CREATE TABLE before AS FROM 'pool_nodes_before.csv';
43+
CREATE TABLE after AS FROM 'pool_nodes_after.csv';
44+
SELECT (CAST(after.select_cnt AS INTEGER) - CAST(before.select_cnt AS INTEGER)) AS diff
45+
FROM before JOIN after USING(node_id)
46+
WHERE node_id = '1';
47+
" | tail -n 1 | tee select_cnt_diff.txt
48+
49+
# Verify that select_cnt increased by 1
50+
if grep -q '^1$' select_cnt_diff.txt; then
51+
echo 'HTAP setup verification successful.'
52+
else
53+
echo 'HTAP setup verification failed.'
54+
exit 1
55+
fi
56+
57+
- name: Cleanup
58+
if: always()
59+
run: |
60+
cd devtools/htap-setup-pg/pgpool2
61+
docker compose down

‎devtools/htap-setup-mysql/maxscale/docker-compose.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ services:
2121
container_name: htap-mysql-setup
2222
command: >
2323
sh -c "
24-
mysql -h mysql -u root -e \"create database test;\"
25-
mysql -h mysql -u root -e \"create table test.test (id bigint primary key, s varchar(100), i int);\"
26-
mysql -h mysql -u root -e \"insert into test.test values (1, 'a', 11), (2, 'b', 22);\"
24+
mysql -h mysql -u root -e \"create database db01;\"
25+
mysql -h mysql -u root -e \"create table db01.test (id bigint primary key, s varchar(100), i int);\"
26+
mysql -h mysql -u root -e \"insert into db01.test values (1, 'a', 11), (2, 'b', 22);\"
2727
"
2828
restart: "on-failure"
2929
depends_on:

‎devtools/htap-setup-mysql/proxysql/docker-compose.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ services:
2121
container_name: htap-mysql-setup
2222
command: >
2323
sh -c "
24-
mysql -h mysql -u root -e \"create database test;\"
25-
mysql -h mysql -u root -e \"create table test.test (id bigint primary key, s varchar(100), i int);\"
26-
mysql -h mysql -u root -e \"insert into test.test values (1, 'a', 11), (2, 'b', 22);\"
24+
mysql -h mysql -u root -e \"create database db01;\"
25+
mysql -h mysql -u root -e \"create table db01.test (id bigint primary key, s varchar(100), i int);\"
26+
mysql -h mysql -u root -e \"insert into db01.test values (1, 'a', 11), (2, 'b', 22);\"
2727
"
2828
restart: "on-failure"
2929
depends_on:

‎docs/tutorial/mysql-htap-maxscale-setup.md

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,15 @@ This a tutorial to build an HTAP service based on MySQL, MyDuck Server, and Mari
22

33
# Prerequisites
44

5-
* Install `docker-compose`
6-
* On MacOS, please run `brew install docker-compose`.
7-
* On Linux, please do the following:
8-
* Run `sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose`.
9-
* And then run `sudo chmod +x /usr/local/bin/docker-compose`.
5+
* Install [Docker Compose](https://docs.docker.com/compose/install/)
106

117
# Launch the HTAP cluster
128

139
Go the root path of this project and run the following commands:
1410

1511
```sh
1612
cd devtools/htap-setup-mysql/maxscale
17-
docker-compose up -d
13+
docker compose up -d --wait
1814
```
1915

2016
Then you'll get a HTAP cluster. And an account 'lol' with password 'lol' has been created for connecting. Have fun!
@@ -60,4 +56,4 @@ after executing the `READ` statement:
6056

6157
# Cleanup
6258

63-
You can run `docker-compose down` to clean up all resources after the trial.
59+
You can run `docker compose down` to clean up all resources after the trial.

‎docs/tutorial/mysql-htap-proxysql-setup.md

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,15 @@ This is a tutorial to build an HTAP service based on MySQL, MyDuck Server, and P
33

44
# Prerequisites
55

6-
* Install `docker-compose`
7-
* On MacOS, please run `brew install docker-compose`.
8-
* On Linux, please do the following:
9-
* Run `sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose`.
10-
* And then run `sudo chmod +x /usr/local/bin/docker-compose`.
6+
* Install [Docker Compose](https://docs.docker.com/compose/install/)
117

128
# Launch the HTAP cluster
139

1410
Go the root path of this project and run the following commands:
1511

16-
```
12+
```sh
1713
cd devtools/htap-setup-mysql/proxysql
18-
docker-compose up -d
14+
docker compose up -d --wait
1915
```
2016

2117
Then you'll get a HTAP cluster. And an account 'lol' with password 'lol' has been created for connecting. Have fun!
@@ -50,4 +46,4 @@ SELECT * FROM stats.stats_mysql_commands_counters;
5046

5147
# Cleanup
5248

53-
You can run `docker-compose down` to clean up all resources after the trial.
49+
You can run `docker compose down` to clean up all resources after the trial.

‎docs/tutorial/pg-htap-pgpool-setup.md

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,15 @@ This a tutorial to build an HTAP service based on PostgreSQL, MyDuck Server, and
22

33
# Prerequisites
44

5-
* Install `docker-compose`
6-
* On MacOS, please run `brew install docker-compose`.
7-
* On Linux, please do the following:
8-
* Run `sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose`.
9-
* And then run `sudo chmod +x /usr/local/bin/docker-compose`.
5+
* Install [Docker Compose](https://docs.docker.com/compose/install/)
106

117
# Launch the HTAP cluster
128

139
Go the root path of this project and run the following commands:
1410

1511
```sh
1612
cd devtools/htap-setup-pg/pgpool2
17-
docker-compose up -d
13+
docker compose up -d --wait
1814
```
1915

2016
Then you'll get a HTAP cluster. Have fun!
@@ -109,4 +105,4 @@ postgres=# show pool_nodes;
109105

110106
# Cleanup
111107

112-
You can run `docker-compose down` to clean up all resources after the trial.
108+
You can run `docker compose down` to clean up all resources after the trial.

0 commit comments

Comments
 (0)