Run more test cases per query fuzz test (#6311)

* run more test cases per query fuzz test

Signed-off-by: Ben Ye <[email protected]>

* run 1000 test cases

Signed-off-by: Ben Ye <[email protected]>

* add retry to EOF errors

Signed-off-by: Ben Ye <[email protected]>

---------

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

Diff for: integration/e2ecortex/client.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
@@ -74,7 +75,7 @@ func NewClient(
 		querierAddress:      querierAddress,
 		alertmanagerAddress: alertmanagerAddress,
 		rulerAddress:        rulerAddress,
-		timeout:             5 * time.Second,
+		timeout:             30 * time.Second,
 		httpClient:          &http.Client{},
 		querierClient:       promv1.NewAPI(querierAPIClient),
 		orgID:               orgID,
@@ -105,7 +106,7 @@ func NewPromQueryClient(address string) (*Client, error) {
 	}
 
 	c := &Client{
-		timeout:       5 * time.Second,
+		timeout:       30 * time.Second,
 		httpClient:    &http.Client{},
 		querierClient: promv1.NewAPI(querierAPIClient),
 	}
@@ -332,7 +333,26 @@ func (c *Client) OTLP(timeseries []prompb.TimeSeries) (*http.Response, error) {
 
 // Query runs an instant query.
 func (c *Client) Query(query string, ts time.Time) (model.Value, error) {
-	value, _, err := c.querierClient.Query(context.Background(), query, ts)
+	ctx := context.Background()
+	retries := backoff.New(ctx, backoff.Config{
+		MinBackoff: 1 * time.Second,
+		MaxBackoff: 3 * time.Second,
+		MaxRetries: 5,
+	})
+	var (
+		value model.Value
+		err   error
+	)
+	for retries.Ongoing() {
+		value, _, err = c.querierClient.Query(context.Background(), query, ts)
+		if err == nil {
+			break
+		}
+		if !strings.Contains(err.Error(), "EOF") {
+			break
+		}
+		retries.Wait()
+	}
 	return value, err
 }
 
@@ -345,11 +365,31 @@ func (c *Client) QueryExemplars(query string, start, end time.Time) ([]promv1.Ex
 
 // QueryRange runs a query range.
 func (c *Client) QueryRange(query string, start, end time.Time, step time.Duration) (model.Value, error) {
-	value, _, err := c.querierClient.QueryRange(context.Background(), query, promv1.Range{
-		Start: start,
-		End:   end,
-		Step:  step,
+	ctx := context.Background()
+	retries := backoff.New(ctx, backoff.Config{
+		MinBackoff: 1 * time.Second,
+		MaxBackoff: 3 * time.Second,
+		MaxRetries: 5,
 	})
+	var (
+		value model.Value
+		err   error
+	)
+	for retries.Ongoing() {
+		value, _, err = c.querierClient.QueryRange(context.Background(), query, promv1.Range{
+			Start: start,
+			End:   end,
+			Step:  step,
+		})
+		if err == nil {
+			break
+		}
+		if !strings.Contains(err.Error(), "EOF") {
+			break
+		}
+		retries.Wait()
+	}
+
 	return value, err
 }
 

Diff for: integration/query_fuzz_test.go

@@ -515,7 +515,7 @@ func TestVerticalShardingFuzz(t *testing.T) {
 	}
 	ps := promqlsmith.New(rnd, lbls, opts...)
 
-	runQueryFuzzTestCases(t, ps, c1, c2, now, start, end, scrapeInterval, 100)
+	runQueryFuzzTestCases(t, ps, c1, c2, now, start, end, scrapeInterval, 1000)
 }
 
 // comparer should be used to compare promql results between engines.
@@ -1065,7 +1065,7 @@ func TestBackwardCompatibilityQueryFuzz(t *testing.T) {
 	}
 	ps := promqlsmith.New(rnd, lbls, opts...)
 
-	runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 100)
+	runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 1000)
 }
 
 // TestPrometheusCompatibilityQueryFuzz compares Cortex with latest Prometheus release.
@@ -1178,7 +1178,7 @@ func TestPrometheusCompatibilityQueryFuzz(t *testing.T) {
 	}
 	ps := promqlsmith.New(rnd, lbls, opts...)
 
-	runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 100)
+	runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 1000)
 }
 
 // waitUntilReady is a helper function to wait and check if both servers to test load the expected data.
@@ -1201,8 +1201,11 @@ func waitUntilReady(t *testing.T, ctx context.Context, c1, c2 *e2ecortex.Client,
 		labelSet2, err = c2.Series([]string{query}, start, end)
 		require.NoError(t, err)
 
-		if cmp.Equal(labelSet1, labelSet2, labelSetsComparer) {
-			break
+		// Make sure series can be queried.
+		if len(labelSet1) > 0 {
+			if cmp.Equal(labelSet1, labelSet2, labelSetsComparer) {
+				break
+			}
 		}
 
 		retries.Wait()