Querying from timestream with pagination is very slow

[Noy-Simon]

When i use the Go client to query a result of 200 rows it takes about 20 seconds
In the process, it paginates 4 times
page 1 - 0 rows returned
page 2 - 0 rows returned
page 3 - 0 rows returned
page 4 -200 rows returned
i don't understand why this is happening
each page returns after ~5 seconds so its no a timeout issue

when i use the timestream query console on amazon the same query takes 2-3 seconds
so why is it 10X slower using the SDK ?

[RanVaknin]

Hi @Noy-Simon ,

Without seeing your code it will be difficult to know why you are running into this.

Thanks,
Ran~

[Noy-Simon]

Hey @RanVaknin this is the code we use to query
tell me if you need anything more

func (c *QueryClient) Query(ctx context.Context, queryString string, pageSize *int, paginationToken *string) (*QueryResponse, error) {
	var maxRows int32 = MaxRows
	if pageSize != nil {
		maxRows = int32(math.Min(float64(*pageSize), MaxRows))
	}

	input := &timestreamquery.QueryInput{
		QueryString: aws.String(queryString),
		MaxRows:     aws.Int32(maxRows),
		NextToken:   toRequestPaginationToken(paginationToken),
	}

	paginator := c.GetPaginator(input)

	return c.QueryInt(ctx, paginator, pageSize)
}

func (c *QueryClient) QueryInt(ctx context.Context, paginator PaginationExecutor, pageSize *int) (*QueryResponse, error) {
	rows := make([]types.Row, 0)
	var totalRows int32 = 0
	var responsePaginationToken *string = nil

	shouldCheckPagination := func(pageSize *int, totalRows int32) bool {
		return pageSize == nil || (pageSize != nil && totalRows < int32(*pageSize))
	}

	for paginator.HasMorePages() && shouldCheckPagination(pageSize, totalRows) {
		output, err := paginator.NextPage(ctx)
		if err != nil {
			return nil, fmt.Errorf("unable to get next page, err: %w", err)
		}

		responsePaginationToken = output.NextToken
		rows = append(rows, output.Rows...)
		totalRows += int32(len(rows))
	}
	return &QueryResponse{
		Rows:            rows,
		PaginationToken: toResponsePaginationToken(responsePaginationToken),
	}, nil
}

[github-actions[bot]]

Hello! Reopening this discussion to make it searchable.