Support copying from glob patterns

Closes #112.

Author: aykut-bozkurt

.devcontainer/.env

@@ -20,7 +20,7 @@ AZURE_TEST_READ_WRITE_SAS="se=2100-05-05&sp=rcw&sv=2022-11-02&sr=c&sig=TPz2jEz0t
 
 # http(s) tests
 ALLOW_HTTP=true
-HTTP_ENDPOINT=http://localhost:8080
+HTTP_ENDPOINT=http://localhost:9999
 
 # GCS tests
 GOOGLE_TEST_BUCKET=testbucket

.devcontainer/docker-compose.yml

@@ -53,13 +53,13 @@ services:
 
   webdav:
     image: rclone/rclone
-    command: ["serve", "webdav", "/data", "--addr", ":8080"]
+    command: ["serve", "webdav", "/data", "--addr", ":9999"]
     env_file:
       - .env
     network_mode: host
     restart: unless-stopped
     healthcheck:
-      test: ["CMD", "nc", "-z", "localhost", "8080"]
+      test: ["CMD", "nc", "-z", "localhost", "9999"]
       interval: 6s
       timeout: 2s
       retries: 3

.github/workflows/ci.yml

@@ -146,7 +146,7 @@ jobs:
         run: |
           docker run -d \
             --env-file .devcontainer/.env \
-            -p 8080:80 \
+            -p 9999:80 \
             rclone/rclone serve webdav /data --addr :80
 
           while ! curl $HTTP_ENDPOINT; do

Cargo.lock

@@ -27,6 +27,7 @@ aws-config = { version = "=1.5.18", default-features = false, features = ["rustl
 aws-credential-types = {version = "=1.2.1", default-features = false}
 azure_storage = {version = "0.21", default-features = false}
 futures = "0.3"
+glob = "0.3"
 home = "0.5"
 object_store = {version = "0.12", default-features = false, features = ["aws", "azure", "fs", "gcp", "http"]}
 once_cell = "1"

Cargo.toml

@@ -22,6 +22,7 @@ COPY table FROM 's3://mybucket/data.parquet' WITH (format 'parquet');
   - [Inspect Parquet schema](#inspect-parquet-schema)
   - [Inspect Parquet metadata](#inspect-parquet-metadata)
   - [Inspect Parquet column statistics](#inspect-parquet-column-statistics)
+  - [List and read Parquet files from uri pattern](#list-and-read-parquet-files-from-uri-pattern)
 - [Object Store Support](#object-store-support)
 - [Copy Options](#copy-options)
 - [Configuration](#configuration)
@@ -192,6 +193,40 @@ SELECT * FROM parquet.column_stats('/tmp/product_example.parquet')
 (13 rows)
 ```
 
+### List and read Parquet files from uri pattern
+
+You can call `SELECT * FROM parquet.list(<uri_pattern>)` to see all uris that matches with the uri pattern.
+Uri pattern can resolve `**` for directories and `*` for words in the uri.
+
+
+```sql
+COPY (SELECT i FROM generate_series(1, 1000000) i) TO '/tmp/some/test.parquet' with (file_size_bytes '1MB');
+COPY 1000000
+
+SELECT * FROM parquet.list('/tmp/some/**/*.parquet');
+                  uri                  |  size
+---------------------------------------+---------
+ /tmp/some/test.parquet/data_4.parquet |  100162
+ /tmp/some/test.parquet/data_3.parquet | 1486916
+ /tmp/some/test.parquet/data_2.parquet | 1486916
+ /tmp/some/test.parquet/data_0.parquet | 1486920
+ /tmp/some/test.parquet/data_1.parquet | 1486916
+(5 rows)
+
+```
+
+Uri pattern is also supported by `COPY FROM` for all supported object stores except `http(s)` endpoints.
+```sql
+COPY (SELECT i FROM generate_series(1, 1000000) i) TO 's3://testbucket/some/test.parquet' with (file_size_bytes '1MB');
+COPY 1000000
+
+CREATE TABLE test(a int);
+CREATE TABLE
+
+COPY test FROM 's3://testbucket/some/**/*.parquet';
+COPY 1000000
+```
+
 ## Object Store Support
 `pg_parquet` supports reading and writing Parquet files from/to `S3`, `Azure Blob Storage`, `http(s)` and `Google Cloud Storage` object stores.
 
@@ -278,7 +313,7 @@ Supported authorization methods' priority order is shown below:
 
 #### Http(s) Storage
 
-`Https` uris are supported by default. You can set `ALLOW_HTTP` environment variable to allow `http` uris.
+Only `Https` uris are supported by default. You can set `ALLOW_HTTP` environment variable to allow `http` uris.
 
 #### Google Cloud Storage