Go Proxy Client With Retry and Circuit Breaker

Go's net/http client is already one of the best HTTP clients shipped with a standard library. It gives you keep-alive, HTTP/2, proxy support via Transport.Proxy, and a sane default connection pool. What it does not give you is intelligent retry, per-endpoint circuit breaking, or load-aware endpoint selection. This guide adds those, using github.com/sony/gobreaker for the breaker state machine.

The result is a proxy client that: picks the least-loaded endpoint per request, retries with exponential backoff on 5xx/429, opens a circuit breaker after 5 consecutive failures, and respects context.Context cancellation throughout.

Module setup

// go.mod
module github.com/example/hexproxy-client

go 1.22

require github.com/sony/gobreaker v1.0.0

gobreaker v1.0.0 is stable and has zero dependencies. Resist the urge to wire up Resilience4j-style configuration libraries — the whole point of Go's approach is that a few hundred lines of explicit code beat magic.

Endpoint state

Each endpoint owns (a) its own *http.Client with a proxy-configured Transport so connection pools do not collide, (b) a gobreaker instance, and (c) an atomic in-flight counter for load-aware picking. Sharing a single Transport across proxies would share the connection pool too, which defeats the purpose.

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"math/rand/v2"
	"net/http"
	"net/url"
	"sync"
	"sync/atomic"
	"time"

	"github.com/sony/gobreaker"
)

// ProxyEndpoint is an immutable proxy description.
type ProxyEndpoint struct {
	URL   string // "http://user:pass@gate.hexproxies.com:7777"
	Label string
}

// endpointState pairs a proxied http.Client with its circuit breaker.
type endpointState struct {
	ep      ProxyEndpoint
	client  *http.Client
	breaker *gobreaker.CircuitBreaker
	inFlight atomic.Int32
}

func newEndpointState(ep ProxyEndpoint) (*endpointState, error) {
	pu, err := url.Parse(ep.URL)
	if err != nil {
		return nil, fmt.Errorf("parse proxy url: %w", err)
	}
	transport := &http.Transport{
		Proxy:               http.ProxyURL(pu),
		MaxIdleConns:        100,
		MaxIdleConnsPerHost: 25,
		IdleConnTimeout:     90 * time.Second,
		ForceAttemptHTTP2:   true,
	}
	cb := gobreaker.NewCircuitBreaker(gobreaker.Settings{
		Name:        ep.Label,
		MaxRequests: 3,
		Interval:    60 * time.Second,
		Timeout:     30 * time.Second,
		ReadyToTrip: func(counts gobreaker.Counts) bool {
			return counts.ConsecutiveFailures >= 5
		},
	})
	return &endpointState{
		ep:      ep,
		client:  &http.Client{Transport: transport, Timeout: 20 * time.Second},
		breaker: cb,
	}, nil
}

Pool with load-aware picking

The pick method iterates all endpoints and returns the one with the fewest in-flight requests, skipping any whose breaker is open. This is an O(N) operation but N is small (typically 2-20 gateways), so the simpler algorithm is the right one. If you have hundreds of endpoints, switch to a min-heap.

// ProxyPool fans requests across a set of proxy endpoints.
type ProxyPool struct {
	mu    sync.RWMutex
	nodes []*endpointState
}

func NewProxyPool(eps []ProxyEndpoint) (*ProxyPool, error) {
	if len(eps) == 0 {
		return nil, errors.New("at least one endpoint required")
	}
	nodes := make([]*endpointState, 0, len(eps))
	for _, ep := range eps {
		s, err := newEndpointState(ep)
		if err != nil {
			return nil, err
		}
		nodes = append(nodes, s)
	}
	return &ProxyPool{nodes: nodes}, nil
}

// pick returns the endpoint with the fewest in-flight requests that is
// not in an open-circuit state.
func (p *ProxyPool) pick() (*endpointState, error) {
	p.mu.RLock()
	defer p.mu.RUnlock()

	var best *endpointState
	bestLoad := int32(1 << 30)
	for _, n := range p.nodes {
		if n.breaker.State() == gobreaker.StateOpen {
			continue
		}
		load := n.inFlight.Load()
		if load < bestLoad {
			best = n
			bestLoad = load
		}
	}
	if best == nil {
		return nil, errors.New("all endpoints unavailable")
	}
	return best, nil
}

// Do executes the request with retry and circuit-break semantics.
// maxAttempts includes the initial try.
func (p *ProxyPool) Do(
	ctx context.Context,
	req *http.Request,
	maxAttempts int,
) (*http.Response, error) {
	var lastErr error
	for attempt := 1; attempt <= maxAttempts; attempt++ {
		node, err := p.pick()
		if err != nil {
			lastErr = err
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			case <-time.After(backoff(attempt)):
				continue
			}
		}

		node.inFlight.Add(1)
		result, cbErr := node.breaker.Execute(func() (any, error) {
			resp, httpErr := node.client.Do(req.Clone(ctx))
			if httpErr != nil {
				return nil, httpErr
			}
			if resp.StatusCode == 429 || resp.StatusCode >= 500 {
				// Drain and close body so the connection can be reused.
				io.Copy(io.Discard, resp.Body)
				_ = resp.Body.Close()
				return nil, fmt.Errorf("retryable status %d", resp.StatusCode)
			}
			return resp, nil
		})
		node.inFlight.Add(-1)

		if cbErr == nil {
			return result.(*http.Response), nil
		}
		lastErr = cbErr

		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-time.After(backoff(attempt)):
		}
	}
	return nil, fmt.Errorf("all attempts failed: %w", lastErr)
}

func backoff(attempt int) time.Duration {
	base := time.Duration(1<<attempt) * 100 * time.Millisecond
	if base > 5*time.Second {
		base = 5 * time.Second
	}
	jitter := time.Duration(rand.Int64N(int64(base / 4)))
	return base + jitter
}

Driving the pool

The main function fires 500 concurrent requests through a 50-wide semaphore. The pool handles retries internally, so the caller only sees a final success or a wrapped error.

func main() {
	pool, err := NewProxyPool([]ProxyEndpoint{
		{URL: "http://USER:PASS@gate.hexproxies.com:7777", Label: "hex-us"},
		{URL: "http://USER:PASS@gate-eu.hexproxies.com:7777", Label: "hex-eu"},
	})
	if err != nil {
		panic(err)
	}

	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()

	var wg sync.WaitGroup
	sem := make(chan struct{}, 50) // global concurrency cap

	for i := 0; i < 500; i++ {
		wg.Add(1)
		sem <- struct{}{}
		go func(i int) {
			defer wg.Done()
			defer func() { <-sem }()

			req, _ := http.NewRequestWithContext(
				ctx, "GET",
				fmt.Sprintf("https://httpbin.org/ip?n=%d", i),
				nil,
			)
			resp, err := pool.Do(ctx, req, 4)
			if err != nil {
				fmt.Printf("req %d: %v\n", i, err)
				return
			}
			_, _ = io.Copy(io.Discard, resp.Body)
			_ = resp.Body.Close()
		}(i)
	}
	wg.Wait()
}

Circuit breaker tuning

The defaults — ConsecutiveFailures >= 5, 30-second timeout — are reasonable for residential proxies, where transient failures are common. For ISP proxies where failures indicate real problems, drop the threshold to 3 and extend the timeout to 2 minutes. Do not set the threshold to 1: you will flip the breaker on single-request hiccups and waste half your capacity.

What this client does not do

It does not retry on idempotent methods only — every request is retried. If you call POST /orders with this pool you could double-submit. Fix that by checking req.Method inside Do() and returning immediately on non-idempotent failures, or by requiring an Idempotency-Key header on writes. The scraping use case doesn't need this because reads are always safe to retry.

It also does not rotate session identity per request. For that, generate a per-session username (USER-session-N) and build a new endpoint per session. Hex Proxies gateway supports this natively. See the ISP vs residential guide to pick the right product.