diff --git a/app/dispatcher/README.md b/app/dispatcher/README.md new file mode 100644 index 000000000..69b7d12a2 --- /dev/null +++ b/app/dispatcher/README.md @@ -0,0 +1,49 @@ +# Waiting for TLS before routing + +An IP-based TCP connection can arrive before its application data. With ordinary +sniffing, an expired sniffing timeout falls back to routing the original IP. A +ClientHello sent later cannot change that route. + +For an inbound whose IP TCP/443 connections must wait for content classification, +enable `tlsSniffingTimeout` in its `sniffing` settings: + +```json +{ + "enabled": true, + "destOverride": ["http", "tls"], + "routeOnly": true, + "tlsSniffingTimeout": 3000 +} +``` + +The value is milliseconds, from 1 to 60000. Zero (the default) retains the ordinary +sniffing timeout policy. Nonzero values require `enabled: true`, `metadataOnly: +false`, and `tls` in `destOverride` (`https` and `ssl` are JSON aliases). + +For IP TCP/443 destinations, Xray waits before dispatching to any outbound. The +single deadline covers both waiting for the first byte and completing sniffing; +receiving another fragment does not restart it. ClientHello fragmentation across +reads and TLS records is supported. Classification finishes as soon as possible, +without adding a fixed delay to a complete ClientHello. + +An expired deadline, cancellation, EOF before classification, an incomplete +64 KiB prefix, or exhaustion of the 1024 pending-sniff slots closes the connection +without dispatching it. The slot limit is per dispatcher, shared by enabled +inbounds. A pending read retains its slot until the inbound owner closes it and +the read returns. SOCKS success still precedes waiting for application data. + +A recognized domain uses existing routing rules and sniffing exclusions. Complete +unrecognized input (including TLS without SNI) retains existing routing behavior; +this setting does not send unknown domains to a different outbound. A FakeDNS +mapping that the sniffing configuration allows to override the destination uses +the ordinary policy, since its domain is already known. Other ports, UDP, and +destinations already supplied as domain names also use the ordinary policy. + +The TLS parser improvements apply with or without this option: fragmented TLS can +continue within the existing 200 ms budget instead of giving up after two reads. +The default timeout and its IP routing fallback remain unchanged. + +Enable this only where waiting for client data is appropriate: a server-first +protocol on TCP/443 will time out. A ClientHello exceeding the prefix budget or +arriving after the deadline will also fail. This is not a guarantee of routing +encrypted/hidden SNI (ECH), QUIC, or all traffic for a particular service. diff --git a/app/dispatcher/default.go b/app/dispatcher/default.go index cae973517..3686180d0 100644 --- a/app/dispatcher/default.go +++ b/app/dispatcher/default.go @@ -4,6 +4,7 @@ import ( "context" "strings" "sync" + "sync/atomic" "time" "github.com/xtls/xray-core/common" @@ -28,8 +29,9 @@ var errSniffingTimeout = errors.New("timeout on sniffing") type cachedReader struct { sync.Mutex - reader buf.TimeoutReader // *pipe.Reader or *buf.TimeoutWrapperReader - cache buf.MultiBuffer + reader buf.TimeoutReader // *pipe.Reader or *buf.TimeoutWrapperReader + cache buf.MultiBuffer + readErr error } func (r *cachedReader) Cache(b *buf.Buffer, deadline time.Duration) error { @@ -67,6 +69,9 @@ func (r *cachedReader) ReadMultiBuffer() (buf.MultiBuffer, error) { if mb != nil { return mb, nil } + if r.readErr != nil { + return nil, r.readErr + } return r.reader.ReadMultiBuffer() } @@ -76,6 +81,9 @@ func (r *cachedReader) ReadMultiBufferTimeout(timeout time.Duration) (buf.MultiB if mb != nil { return mb, nil } + if r.readErr != nil { + return nil, r.readErr + } return r.reader.ReadMultiBufferTimeout(timeout) } @@ -93,11 +101,12 @@ func (r *cachedReader) Interrupt() { // DefaultDispatcher is a default implementation of Dispatcher. type DefaultDispatcher struct { - ohm outbound.Manager - router routing.Router - policy policy.Manager - stats stats.Manager - fdns dns.FakeDNSEngine + pendingTLSSniffs atomic.Int32 + ohm outbound.Manager + router routing.Router + policy policy.Manager + stats stats.Manager + fdns dns.FakeDNSEngine } func init() { @@ -292,7 +301,13 @@ func (d *DefaultDispatcher) Dispatch(ctx context.Context, destination net.Destin reader: outbound.Reader.(*pipe.Reader), } outbound.Reader = cReader - result, err := sniffer(ctx, cReader, sniffingRequest.MetadataOnly, destination.Network) + result, err := d.sniff(ctx, cReader, sniffingRequest, destination) + if errors.Cause(err) == errSniffingRejected { + errors.LogInfoInner(ctx, err, "closing connection before routing") + common.Close(outbound.Writer) + cReader.Interrupt() + return + } if err == nil { content.Protocol = result.Protocol() } @@ -347,7 +362,12 @@ func (d *DefaultDispatcher) DispatchLink(ctx context.Context, destination net.De reader: outbound.Reader.(buf.TimeoutReader), } outbound.Reader = cReader - result, err := sniffer(ctx, cReader, sniffingRequest.MetadataOnly, destination.Network) + result, err := d.sniff(ctx, cReader, sniffingRequest, destination) + if errors.Cause(err) == errSniffingRejected { + common.Close(outbound.Writer) + cReader.Interrupt() + return err + } if err == nil { content.Protocol = result.Protocol() } diff --git a/app/dispatcher/tls_sniff.go b/app/dispatcher/tls_sniff.go new file mode 100644 index 000000000..e0fca92da --- /dev/null +++ b/app/dispatcher/tls_sniff.go @@ -0,0 +1,121 @@ +package dispatcher + +import ( + "context" + "io" + + "github.com/xtls/xray-core/common" + "github.com/xtls/xray-core/common/buf" + "github.com/xtls/xray-core/common/errors" + "github.com/xtls/xray-core/common/net" + "github.com/xtls/xray-core/common/protocol" + "github.com/xtls/xray-core/common/session" +) + +const ( + maxPendingTLSSniffs = 1024 + maxTLSSniffBytes = 64 * 1024 +) + +var errSniffingRejected = errors.New("TLS sniffing incomplete; refusing to route") + +type tlsSniffResult struct { + result SniffResult + err error + cache buf.MultiBuffer + readErr error +} + +func (d *DefaultDispatcher) sniff(ctx context.Context, reader *cachedReader, request session.SniffingRequest, destination net.Destination) (SniffResult, error) { + if request.TLSSniffingTimeout == 0 || destination.Network != net.Network_TCP || destination.Port != 443 || !destination.Address.Family().IsIP() { + return sniffer(ctx, reader, request.MetadataOnly, destination.Network) + } + // A usable FakeDNS mapping already supplies the domain needed for routing. + if metadata, err := NewSniffer(ctx).SniffMetadata(ctx); err == nil && d.shouldOverride(ctx, metadata, request, destination) { + return sniffer(ctx, reader, request.MetadataOnly, destination.Network) + } + if d.pendingTLSSniffs.Add(1) > maxPendingTLSSniffs { + d.pendingTLSSniffs.Add(-1) + return nil, errors.New("too many pending TLS sniffs").Base(errSniffingRejected) + } + ctx, cancel := context.WithTimeout(ctx, request.TLSSniffingTimeout) + defer cancel() + done := make(chan tlsSniffResult) + go func() { + // Retain the slot until the reader exits, even if its owner is slow to close it. + defer d.pendingTLSSniffs.Add(-1) + result := readTLSSniff(ctx, reader.reader) + defer func() { buf.ReleaseMulti(result.cache) }() + select { + case done <- result: + result.cache = nil + case <-ctx.Done(): + } + }() + select { + case <-ctx.Done(): + return nil, errors.New(ctx.Err()).Base(errSniffingRejected) + case result := <-done: + if ctx.Err() != nil { + buf.ReleaseMulti(result.cache) + return nil, errors.New(ctx.Err()).Base(errSniffingRejected) + } + reader.cache = result.cache + reader.readErr = result.readErr + return result.result, result.err + } +} + +func readTLSSniff(ctx context.Context, reader buf.Reader) (out tlsSniffResult) { + sniffer := NewSniffer(ctx) + metadata, metadataErr := sniffer.SniffMetadata(ctx) + var payload []byte + emptyReads := 0 + for { + if ctx.Err() != nil { + out.err = errSniffingRejected + return + } + mb, readErr := reader.ReadMultiBuffer() + out.readErr = readErr + if mb.IsEmpty() { + emptyReads++ + } else { + emptyReads = 0 + } + for _, b := range mb { + if b == nil { + continue + } + payload = append(payload, b.BytesTo(min(b.Len(), int32(maxTLSSniffBytes-len(payload))))...) + // Coalesce small reads so a byte-at-a-time peer cannot retain one buffer per byte. + out.cache = buf.MergeBytes(out.cache, b.Bytes()) + } + buf.ReleaseMulti(mb) + if len(payload) > 0 { + result, err := sniffer.Sniff(ctx, payload, net.Network_TCP) + if err != common.ErrNoClue && err != protocol.ErrProtoNeedMoreData { + if metadataErr == nil { + if err == nil { + result = CompositeResult(metadata, result) + } else { + result, err = metadata, nil + } + } + out.result, out.err = result, err + return + } + } + if readErr != nil || len(payload) == maxTLSSniffBytes || emptyReads >= 64 { + if readErr == nil { + if len(payload) == maxTLSSniffBytes { + readErr = errors.New("TLS sniffing prefix limit reached") + } else { + readErr = io.ErrNoProgress + } + } + out.err = errors.New(readErr).Base(errSniffingRejected) + return + } + } +} diff --git a/app/dispatcher/tls_sniff_test.go b/app/dispatcher/tls_sniff_test.go new file mode 100644 index 000000000..d013dc96b --- /dev/null +++ b/app/dispatcher/tls_sniff_test.go @@ -0,0 +1,365 @@ +package dispatcher + +import ( + "bytes" + "context" + "io" + "os" + "testing" + "time" + + "github.com/xtls/xray-core/common" + "github.com/xtls/xray-core/common/buf" + "github.com/xtls/xray-core/common/errors" + "github.com/xtls/xray-core/common/net" + "github.com/xtls/xray-core/common/session" + "github.com/xtls/xray-core/core" + "github.com/xtls/xray-core/features/dns" + "github.com/xtls/xray-core/features/outbound" + "github.com/xtls/xray-core/features/routing" + "github.com/xtls/xray-core/transport" + "github.com/xtls/xray-core/transport/pipe" +) + +type sniffTestManager struct { + outbound.Manager + handlers map[string]outbound.Handler +} + +func (m *sniffTestManager) GetDefaultHandler() outbound.Handler { return m.handlers["direct"] } +func (m *sniffTestManager) GetHandler(tag string) outbound.Handler { return m.handlers[tag] } + +type ( + sniffTestRouter struct{ routing.DefaultRouter } + sniffTestRoute struct{ routing.Context } +) + +func (sniffTestRoute) GetOutboundTag() string { return "selected" } +func (sniffTestRoute) GetRuleTag() string { return "" } +func (sniffTestRoute) GetOutboundGroupTags() []string { return nil } +func (sniffTestRouter) PickRoute(ctx routing.Context) (routing.Route, error) { + if ctx.GetTargetDomain() == "www.youtube.com" { + return sniffTestRoute{ctx}, nil + } + return nil, common.ErrNoClue +} + +type ( + sniffDispatch struct { + tag string + payload []byte + } + sniffTestHandler struct { + outbound.Handler + tag string + called chan string + finished chan sniffDispatch + } +) + +func (h *sniffTestHandler) Tag() string { return h.tag } +func (h *sniffTestHandler) Dispatch(ctx context.Context, link *transport.Link) { + h.called <- h.tag + var payload []byte + for { + mb, err := link.Reader.ReadMultiBuffer() + for _, b := range mb { + payload = append(payload, b.Bytes()...) + } + buf.ReleaseMulti(mb) + if err != nil { + break + } + } + h.finished <- sniffDispatch{h.tag, payload} + common.Close(link.Writer) +} + +func sniffTestContext(timeout time.Duration) context.Context { + ctx := context.WithValue(context.Background(), core.XrayKey(1), new(core.Instance)) + return session.ContextWithContent(ctx, &session.Content{SniffingRequest: session.SniffingRequest{ + Enabled: true, RouteOnly: true, OverrideDestinationForProtocol: []string{"http", "tls"}, TLSSniffingTimeout: timeout, + }}) +} + +func sniffFixture(t *testing.T, name string) []byte { + t.Helper() + b, err := os.ReadFile("../../common/protocol/tls/testdata/" + name + ".bin") + if err != nil { + t.Fatal(err) + } + return b +} + +func awaitSniffWorkers(t *testing.T, d *DefaultDispatcher) { + t.Helper() + deadline := time.Now().Add(time.Second) + for d.pendingTLSSniffs.Load() != 0 { + if time.Now().After(deadline) { + t.Fatalf("pending sniffs leaked: %d", d.pendingTLSSniffs.Load()) + } + time.Sleep(time.Millisecond) + } +} + +func TestTLSRoutingBeforeDispatch(t *testing.T) { + hello := sniffFixture(t, "clienthello") + other := bytes.ReplaceAll(hello, []byte("www.youtube.com"), []byte("www.example.com")) + noSNI := sniffFixture(t, "clienthello-no-sni") + splitRecords := append([]byte{0x16, 3, 1, 0, 2}, hello[5:7]...) + rest := len(hello) - 7 + splitRecords = append(splitRecords, 0x16, 3, 1, byte(rest>>8), byte(rest)) + splitRecords = append(splitRecords, hello[7:]...) + for _, linkMode := range []bool{false, true} { + mode := "Dispatch" + if linkMode { + mode = "DispatchLink" + } + t.Run(mode, func(t *testing.T) { + for _, tc := range []struct { + name string + payload []byte + delay time.Duration + chunk int + timeout time.Duration + want string + cancel bool + }{ + {"immediate", hello, 0, 0, time.Second, "selected", false}, + {"late", hello, 350 * time.Millisecond, 0, time.Second, "selected", false}, + {"many reads", hello, 0, 1, time.Second, "selected", false}, + {"multiple records", splitRecords, 0, 3, time.Second, "selected", false}, + {"trailing bytes", append(bytes.Clone(hello), []byte("following application data")...), 0, 0, time.Second, "selected", false}, + {"unselected", other, 0, 0, time.Second, "direct", false}, + {"no SNI", noSNI, 0, 0, time.Second, "direct", false}, + {"non TLS", []byte("GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"), 0, 0, time.Second, "direct", false}, + {"EOF", nil, 0, 0, time.Second, "", false}, + {"partial EOF", hello[:10], 0, 0, time.Second, "", false}, + {"idle timeout", nil, 80 * time.Millisecond, 0, 30 * time.Millisecond, "", false}, + {"late beyond timeout", hello, 80 * time.Millisecond, 0, 30 * time.Millisecond, "", false}, + {"cancel", nil, 0, 0, time.Second, "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + called := make(chan string, 1) + finished := make(chan sniffDispatch, 1) + manager := &sniffTestManager{handlers: make(map[string]outbound.Handler)} + for _, tag := range []string{"direct", "selected"} { + manager.handlers[tag] = &sniffTestHandler{tag: tag, called: called, finished: finished} + } + d := &DefaultDispatcher{ohm: manager, router: sniffTestRouter{}} + ctx, cancel := context.WithCancel(sniffTestContext(tc.timeout)) + defer cancel() + dest := net.TCPDestination(net.ParseAddress("192.0.2.1"), 443) + var link *transport.Link + if linkMode { + upR, upW := pipe.New() + downR, downW := pipe.New() + link = &transport.Link{Reader: downR, Writer: upW} + go func() { + d.DispatchLink(ctx, dest, &transport.Link{Reader: upR, Writer: downW}) + // The inbound owns and closes its reader after DispatchLink returns. + upR.Interrupt() + }() + } else { + var err error + link, err = d.Dispatch(ctx, dest) + if err != nil { + t.Fatal(err) + } + } + defer common.Interrupt(link.Reader) + defer common.Close(link.Writer) + if tc.cancel { + cancel() + } + if tc.delay > 0 { + select { + case tag := <-called: + t.Fatalf("outbound %q opened before application data", tag) + case <-time.After(tc.delay): + } + } + chunk := tc.chunk + if chunk == 0 { + chunk = max(1, len(tc.payload)) + } + for i := 0; i < len(tc.payload); i += chunk { + err := link.Writer.WriteMultiBuffer(buf.MergeBytes(nil, tc.payload[i:min(i+chunk, len(tc.payload))])) + if err != nil && tc.want != "" { + t.Fatal(err) + } + } + common.Close(link.Writer) + _, err := link.Reader.(buf.TimeoutReader).ReadMultiBufferTimeout(2 * time.Second) + if err == buf.ErrReadTimeout { + t.Fatal("dispatcher did not finish") + } + if tc.want == "" { + select { + case tag := <-called: + t.Fatalf("rejected flow dispatched to %q", tag) + default: + } + } else { + select { + case result := <-finished: + if result.tag != tc.want || !bytes.Equal(result.payload, tc.payload) { + t.Fatalf("route=%s bytes=%d, want route=%s bytes=%d", result.tag, len(result.payload), tc.want, len(tc.payload)) + } + case <-time.After(time.Second): + t.Fatal("no outbound result") + } + } + awaitSniffWorkers(t, d) + }) + } + }) + } +} + +type sniffReadFunc func() (buf.MultiBuffer, error) + +func (f sniffReadFunc) ReadMultiBuffer() (buf.MultiBuffer, error) { return f() } + +func TestTLSReadOwnership(t *testing.T) { + ctx := sniffTestContext(time.Second) + dest := net.TCPDestination(net.ParseAddress("192.0.2.1"), 443) + request := session.ContentFromContext(ctx).SniffingRequest + t.Run("data and EOF", func(t *testing.T) { + hello := sniffFixture(t, "clienthello") + r := &cachedReader{reader: &buf.TimeoutWrapperReader{Reader: sniffReadFunc(func() (buf.MultiBuffer, error) { return buf.MergeBytes(nil, hello), io.EOF })}} + d := new(DefaultDispatcher) + result, err := d.sniff(ctx, r, request, dest) + if err != nil || result.Domain() != "www.youtube.com" { + t.Fatalf("%v %v", result, err) + } + mb, err := r.ReadMultiBuffer() + defer buf.ReleaseMulti(mb) + got := make([]byte, mb.Len()) + mb.Copy(got) + if err != nil || !bytes.Equal(got, hello) { + t.Fatal("data returned with EOF was lost") + } + if _, err := r.ReadMultiBuffer(); err != io.EOF { + t.Fatalf("terminal error lost: %v", err) + } + awaitSniffWorkers(t, d) + }) + t.Run("cancel outstanding read", func(t *testing.T) { + started, release := make(chan struct{}), make(chan struct{}) + late := buf.New() + late.Write([]byte("late input")) + r := &cachedReader{reader: &buf.TimeoutWrapperReader{Reader: sniffReadFunc(func() (buf.MultiBuffer, error) { close(started); <-release; return buf.MultiBuffer{late}, nil })}} + d := new(DefaultDispatcher) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan error, 1) + go func() { _, err := d.sniff(ctx, r, request, dest); done <- err }() + <-started + cancel() + if err := <-done; errors.Cause(err) != errSniffingRejected { + t.Fatalf("cancel: %v", err) + } + if d.pendingTLSSniffs.Load() != 1 { + t.Fatal("outstanding reader no longer counts toward limit") + } + close(release) + awaitSniffWorkers(t, d) + if !late.IsEmpty() { + t.Fatal("late read buffer not released") + } + }) + t.Run("overload", func(t *testing.T) { + d := new(DefaultDispatcher) + d.pendingTLSSniffs.Store(maxPendingTLSSniffs) + _, err := d.sniff(ctx, nil, request, dest) + if errors.Cause(err) != errSniffingRejected || d.pendingTLSSniffs.Load() != maxPendingTLSSniffs { + t.Fatalf("overload: %v", err) + } + }) + t.Run("nil buffer on EOF", func(t *testing.T) { + r := &cachedReader{reader: &buf.TimeoutWrapperReader{Reader: sniffReadFunc(func() (buf.MultiBuffer, error) { return buf.MultiBuffer{nil}, io.EOF })}} + d := new(DefaultDispatcher) + _, err := d.sniff(ctx, r, request, dest) + defer r.Interrupt() + if errors.Cause(err) != errSniffingRejected { + t.Fatalf("empty EOF: %v", err) + } + awaitSniffWorkers(t, d) + }) + t.Run("prefix limit", func(t *testing.T) { + // A valid record header repeatedly carries a ClientHello advertising a 16 MiB body. + record := append([]byte{0x16, 3, 1, 0x40, 0}, make([]byte, 16384)...) + copy(record[5:], []byte{1, 0xff, 0xff, 0xff}) + r := &cachedReader{reader: &buf.TimeoutWrapperReader{Reader: sniffReadFunc(func() (buf.MultiBuffer, error) { return buf.MergeBytes(nil, record), nil })}} + d := new(DefaultDispatcher) + _, err := d.sniff(ctx, r, request, dest) + defer r.Interrupt() + if errors.Cause(err) != errSniffingRejected { + t.Fatalf("oversized incomplete hello: %v", err) + } + awaitSniffWorkers(t, d) + }) +} + +func TestTLSSniffScope(t *testing.T) { + for _, tc := range []struct { + name string + dest net.Destination + timeout time.Duration + }{ + {"disabled", net.TCPDestination(net.ParseAddress("192.0.2.1"), 443), 0}, + {"other port", net.TCPDestination(net.ParseAddress("192.0.2.1"), 80), time.Second}, + {"known domain", net.TCPDestination(net.ParseAddress("www.example.com"), 443), time.Second}, + {"UDP", net.UDPDestination(net.ParseAddress("192.0.2.1"), 443), time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := sniffTestContext(tc.timeout) + r := &cachedReader{reader: &buf.TimeoutWrapperReader{Reader: sniffReadFunc(func() (buf.MultiBuffer, error) { return nil, io.EOF })}} + d := new(DefaultDispatcher) + _, err := d.sniff(ctx, r, session.ContentFromContext(ctx).SniffingRequest, tc.dest) + if errors.Cause(err) == errSniffingRejected { + t.Fatalf("strict policy escaped its scope: %v", err) + } + if d.pendingTLSSniffs.Load() != 0 { + t.Fatal("non-strict flow used a pending slot") + } + }) + } +} + +type sniffTestFakeDNS struct{ dns.FakeDNSEngine } + +func (sniffTestFakeDNS) Type() interface{} { return (*dns.FakeDNSEngine)(nil) } +func (sniffTestFakeDNS) GetDomainFromFakeDNS(net.Address) string { return "www.youtube.com" } + +func TestTLSSniffKnownFakeDNS(t *testing.T) { + for _, useFakeDNS := range []bool{false, true} { + ctx := sniffTestContext(time.Millisecond) + dest := net.TCPDestination(net.ParseAddress("198.18.0.1"), 443) + ctx = session.ContextWithOutbounds(ctx, []*session.Outbound{{Target: dest}}) + fakeDNS := sniffTestFakeDNS{} + if err := core.MustFromContext(ctx).AddFeature(fakeDNS); err != nil { + t.Fatal(err) + } + request := session.ContentFromContext(ctx).SniffingRequest + if useFakeDNS { + request.OverrideDestinationForProtocol = append(request.OverrideDestinationForProtocol, "fakedns") + } + r, w := pipe.New() + defer r.Interrupt() + defer w.Close() + cached := &cachedReader{reader: r} + d := &DefaultDispatcher{fdns: fakeDNS} + result, err := d.sniff(ctx, cached, request, dest) + if useFakeDNS { + if err != nil || result.Domain() != "www.youtube.com" || !d.shouldOverride(ctx, result, request, dest) { + t.Fatalf("usable FakeDNS mapping lost on timeout: %v %v", result, err) + } + } else if errors.Cause(err) != errSniffingRejected { + t.Fatalf("unused FakeDNS mapping bypassed guard: %v", err) + } + r.Interrupt() + awaitSniffWorkers(t, d) + } +} diff --git a/app/proxyman/config.go b/app/proxyman/config.go index 5d2b2363c..974ffd6bd 100644 --- a/app/proxyman/config.go +++ b/app/proxyman/config.go @@ -1,20 +1,41 @@ package proxyman import ( + "slices" + "time" + + "github.com/xtls/xray-core/common/errors" "github.com/xtls/xray-core/common/geodata" "github.com/xtls/xray-core/common/session" ) +func (c *SniffingConfig) Validate() error { + if c.TlsSniffingTimeout == 0 { + return nil + } + if c.TlsSniffingTimeout > 60000 { + return errors.New("tlsSniffingTimeout must not exceed 60000 milliseconds") + } + if !c.Enabled || c.MetadataOnly || !slices.Contains(c.DestinationOverride, "tls") { + return errors.New("tlsSniffingTimeout requires enabled content sniffing and tls in destOverride") + } + return nil +} + func BuildSniffingRequest(config *SniffingConfig) (session.SniffingRequest, error) { if config == nil { return session.SniffingRequest{}, nil } + if err := config.Validate(); err != nil { + return session.SniffingRequest{}, err + } request := session.SniffingRequest{ Enabled: config.Enabled, OverrideDestinationForProtocol: config.DestinationOverride, MetadataOnly: config.MetadataOnly, RouteOnly: config.RouteOnly, + TLSSniffingTimeout: time.Duration(config.TlsSniffingTimeout) * time.Millisecond, } if len(config.DomainsExcluded) > 0 { excludeForDomain, err := geodata.DomainReg.BuildDomainMatcher(config.DomainsExcluded) diff --git a/app/proxyman/config.pb.go b/app/proxyman/config.pb.go index 37f5c6c39..75f99dcb0 100644 --- a/app/proxyman/config.pb.go +++ b/app/proxyman/config.pb.go @@ -73,10 +73,13 @@ type SniffingConfig struct { // Whether should only try to sniff metadata without waiting for client input. // Can be used to support SMTP like protocol where server send the first // message. - MetadataOnly bool `protobuf:"varint,4,opt,name=metadata_only,json=metadataOnly,proto3" json:"metadata_only,omitempty"` - RouteOnly bool `protobuf:"varint,5,opt,name=route_only,json=routeOnly,proto3" json:"route_only,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + MetadataOnly bool `protobuf:"varint,4,opt,name=metadata_only,json=metadataOnly,proto3" json:"metadata_only,omitempty"` + RouteOnly bool `protobuf:"varint,5,opt,name=route_only,json=routeOnly,proto3" json:"route_only,omitempty"` + // Opt-in deadline (milliseconds) for content sniffing before routing IP TCP/443. + // A timeout or incomplete input closes the connection without dispatching it. + TlsSniffingTimeout uint32 `protobuf:"varint,7,opt,name=tls_sniffing_timeout,json=tlsSniffingTimeout,proto3" json:"tls_sniffing_timeout,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SniffingConfig) Reset() { @@ -151,6 +154,13 @@ func (x *SniffingConfig) GetRouteOnly() bool { return false } +func (x *SniffingConfig) GetTlsSniffingTimeout() uint32 { + if x != nil { + return x.TlsSniffingTimeout + } + return 0 +} + type ReceiverConfig struct { state protoimpl.MessageState `protogen:"open.v1"` // PortList specifies the ports which the Receiver should listen on. @@ -479,7 +489,7 @@ var File_app_proxyman_config_proto protoreflect.FileDescriptor const file_app_proxyman_config_proto_rawDesc = "" + "\n" + "\x19app/proxyman/config.proto\x12\x11xray.app.proxyman\x1a\x18common/net/address.proto\x1a\x15common/net/port.proto\x1a\x1ftransport/internet/config.proto\x1a!common/serial/typed_message.proto\x1a\x1bcommon/geodata/geodat.proto\"\x0f\n" + - "\rInboundConfig\"\xad\x02\n" + + "\rInboundConfig\"\xdf\x02\n" + "\x0eSniffingConfig\x12\x18\n" + "\aenabled\x18\x01 \x01(\bR\aenabled\x121\n" + "\x14destination_override\x18\x02 \x03(\tR\x13destinationOverride\x12J\n" + @@ -487,7 +497,8 @@ const file_app_proxyman_config_proto_rawDesc = "" + "\fips_excluded\x18\x06 \x03(\v2\x1b.xray.common.geodata.IPRuleR\vipsExcluded\x12#\n" + "\rmetadata_only\x18\x04 \x01(\bR\fmetadataOnly\x12\x1d\n" + "\n" + - "route_only\x18\x05 \x01(\bR\trouteOnly\"\xe5\x02\n" + + "route_only\x18\x05 \x01(\bR\trouteOnly\x120\n" + + "\x14tls_sniffing_timeout\x18\a \x01(\rR\x12tlsSniffingTimeout\"\xe5\x02\n" + "\x0eReceiverConfig\x126\n" + "\tport_list\x18\x01 \x01(\v2\x19.xray.common.net.PortListR\bportList\x123\n" + "\x06listen\x18\x02 \x01(\v2\x1b.xray.common.net.IPOrDomainR\x06listen\x12N\n" + diff --git a/app/proxyman/config.proto b/app/proxyman/config.proto index 7a0387147..4aef09d06 100644 --- a/app/proxyman/config.proto +++ b/app/proxyman/config.proto @@ -32,6 +32,10 @@ message SniffingConfig { bool metadata_only = 4; bool route_only = 5; + + // Opt-in deadline (milliseconds) for content sniffing before routing IP TCP/443. + // A timeout or incomplete input closes the connection without dispatching it. + uint32 tls_sniffing_timeout = 7; } message ReceiverConfig { diff --git a/app/proxyman/config_test.go b/app/proxyman/config_test.go new file mode 100644 index 000000000..7771f6a1c --- /dev/null +++ b/app/proxyman/config_test.go @@ -0,0 +1,44 @@ +package proxyman_test + +import ( + "testing" + "time" + + "google.golang.org/protobuf/proto" + + "github.com/xtls/xray-core/app/proxyman" +) + +func TestTLSSniffingConfig(t *testing.T) { + for _, tc := range []struct { + name string + config *proxyman.SniffingConfig + valid bool + }{ + {"default", &proxyman.SniffingConfig{}, true}, + {"enabled", &proxyman.SniffingConfig{Enabled: true, DestinationOverride: []string{"tls"}, TlsSniffingTimeout: 3000}, true}, + {"maximum", &proxyman.SniffingConfig{Enabled: true, DestinationOverride: []string{"tls"}, TlsSniffingTimeout: 60000}, true}, + {"too long", &proxyman.SniffingConfig{Enabled: true, DestinationOverride: []string{"tls"}, TlsSniffingTimeout: 60001}, false}, + {"disabled", &proxyman.SniffingConfig{DestinationOverride: []string{"tls"}, TlsSniffingTimeout: 3000}, false}, + {"metadata only", &proxyman.SniffingConfig{Enabled: true, MetadataOnly: true, DestinationOverride: []string{"tls"}, TlsSniffingTimeout: 3000}, false}, + {"no tls", &proxyman.SniffingConfig{Enabled: true, DestinationOverride: []string{"http"}, TlsSniffingTimeout: 3000}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + wire, err := proto.Marshal(tc.config) + if err != nil { + t.Fatal(err) + } + var config proxyman.SniffingConfig + if err := proto.Unmarshal(wire, &config); err != nil { + t.Fatal(err) + } + r, err := proxyman.BuildSniffingRequest(&config) + if (err == nil) != tc.valid { + t.Fatalf("valid=%v: %v", tc.valid, err) + } + if tc.valid && r.TLSSniffingTimeout != time.Duration(tc.config.TlsSniffingTimeout)*time.Millisecond { + t.Fatalf("timeout lost: %v", r.TLSSniffingTimeout) + } + }) + } +} diff --git a/common/protocol/tls/fragment_test.go b/common/protocol/tls/fragment_test.go new file mode 100644 index 000000000..47ce3e9f8 --- /dev/null +++ b/common/protocol/tls/fragment_test.go @@ -0,0 +1,120 @@ +package tls_test + +import ( + "bytes" + "encoding/binary" + "os" + "testing" + + "github.com/xtls/xray-core/common/protocol" + . "github.com/xtls/xray-core/common/protocol/tls" +) + +func clientHello(t testing.TB) []byte { + t.Helper() + // Generated using Python ssl.SSLContext.wrap_bio, with www.youtube.com as SNI. + b, err := os.ReadFile("testdata/clienthello.bin") + if err != nil { + t.Fatal(err) + } + return b +} + +func record(payload []byte) []byte { + b := []byte{0x16, 3, 1, 0, 0} + binary.BigEndian.PutUint16(b[3:], uint16(len(payload))) + return append(b, payload...) +} + +func checkPrefixes(t *testing.T, b []byte) { + t.Helper() + for n := 1; n < len(b); n++ { + if _, err := SniffTLS(b[:n]); err != protocol.ErrProtoNeedMoreData { + t.Fatalf("prefix %d/%d: %v", n, len(b), err) + } + } + before := bytes.Clone(b) + h, err := SniffTLS(b) + if err != nil || h.Domain() != "www.youtube.com" { + t.Fatalf("complete hello: %v, %v", h, err) + } + if !bytes.Equal(before, b) { + t.Fatal("sniffer modified the forwarded input") + } +} + +func TestTLSFragmentedClientHello(t *testing.T) { + b := clientHello(t) + checkPrefixes(t, b) + for split := 1; split < len(b)-5; split++ { + joined := append(record(b[5:5+split]), record(b[5+split:])...) + checkPrefixes(t, joined) + } + var tinyRecords []byte + for _, v := range b[5:] { + tinyRecords = append(tinyRecords, record([]byte{v})...) + } + checkPrefixes(t, tinyRecords) + // A following handshake/application record must not affect ClientHello parsing. + check := append(bytes.Clone(b[5:]), 0x0b, 0, 0, 0) + h, err := SniffTLS(record(check)) + if err != nil || h.Domain() != "www.youtube.com" { + t.Fatalf("trailing handshake: %v, %v", h, err) + } +} + +func TestTLSCompleteUnknownAndMalformed(t *testing.T) { + noSNI, err := os.ReadFile("testdata/clienthello-no-sni.bin") + if err != nil { + t.Fatal(err) + } + for _, b := range [][]byte{ + noSNI, + []byte("GET / HTTP/1.1\r\n\r\n"), + {0x16, 3, 1, 0, 0}, + {0x16, 3, 1, 0x40, 1}, + record([]byte{2, 0, 0, 0}), + record([]byte{1, 0, 0, 0}), + } { + if _, err := SniffTLS(b); err == nil || err == protocol.ErrProtoNeedMoreData { + t.Fatalf("complete unknown/malformed input classified as valid or incomplete: %x: %v", b, err) + } + } +} + +func TestTLSLargeClientHello(t *testing.T) { + b := clientHello(t) + // Extend this fixture's final padding extension to make a 32 KiB handshake. + const extensionsOffset, paddingOffset = 116, 291 + if binary.BigEndian.Uint16(b[paddingOffset:]) != 21 || paddingOffset+4+int(binary.BigEndian.Uint16(b[paddingOffset+2:])) != len(b) { + t.Fatal("fixture no longer ends in the expected padding extension") + } + delta := 32768 - (len(b) - 5) + binary.BigEndian.PutUint16(b[extensionsOffset:], binary.BigEndian.Uint16(b[extensionsOffset:])+uint16(delta)) + binary.BigEndian.PutUint16(b[paddingOffset+2:], binary.BigEndian.Uint16(b[paddingOffset+2:])+uint16(delta)) + b = append(b, make([]byte, delta)...) + helloLength := len(b) - 9 + b[6], b[7], b[8] = byte(helloLength>>16), byte(helloLength>>8), byte(helloLength) + joined := append(record(b[5:5+16384]), record(b[5+16384:])...) + for _, n := range []int{1, 4, 5, 16388, 16389, 16390, 16394, len(joined) - 1} { + if _, err := SniffTLS(joined[:n]); err != protocol.ErrProtoNeedMoreData { + t.Fatalf("prefix %d: %v", n, err) + } + } + h, err := SniffTLS(joined) + if err != nil || h.Domain() != "www.youtube.com" { + t.Fatalf("large hello: %v %v", h, err) + } +} + +func FuzzSniffTLS(f *testing.F) { + f.Add(clientHello(f)) + f.Add([]byte{0x16, 3, 1, 0, 4, 1, 0, 0, 0}) + f.Fuzz(func(t *testing.T, b []byte) { + before := bytes.Clone(b) + SniffTLS(b) + if !bytes.Equal(before, b) { + t.Fatal("input modified") + } + }) +} diff --git a/common/protocol/tls/sniff.go b/common/protocol/tls/sniff.go index 11660b92b..09cb78fe7 100644 --- a/common/protocol/tls/sniff.go +++ b/common/protocol/tls/sniff.go @@ -129,25 +129,58 @@ func ReadClientHello(data []byte, h *SniffHeader) error { } func SniffTLS(b []byte) (*SniffHeader, error) { - if len(b) < 5 { + if len(b) == 0 { return nil, common.ErrNoClue } - if b[0] != 0x16 /* TLS Handshake */ { return nil, errNotTLS } - if !IsValidTLSVersion(b[1], b[2]) { - return nil, errNotTLS - } - headerLen := int(binary.BigEndian.Uint16(b[3:5])) - if 5+headerLen > len(b) { - return nil, common.ErrNoClue - } - h := &SniffHeader{} - err := ReadClientHello(b[5:5+headerLen], h) - if err == nil { - return h, nil + // A ClientHello can span both reads and TLS records. Allocate only for bytes + // actually received, never for the length advertised by the peer. + var hello []byte + for { + if len(b) < 5 { + return nil, protocol.ErrProtoNeedMoreData + } + if b[0] != 0x16 || !IsValidTLSVersion(b[1], b[2]) { + return nil, errNotTLS + } + recordLen := int(binary.BigEndian.Uint16(b[3:5])) + if recordLen == 0 || recordLen > 16384 { + return nil, errNotClientHello + } + if len(b) < 5+recordLen { + return nil, protocol.ErrProtoNeedMoreData + } + record := b[5 : 5+recordLen] + if hello == nil { + hello = record[:len(record):len(record)] + } else { + hello = append(hello, record...) + } + if hello[0] != 1 { // ClientHello + return nil, errNotClientHello + } + if len(hello) >= 4 { + helloLen := 4 + (int(hello[1]) << 16) + (int(hello[2]) << 8) + int(hello[3]) + if len(hello) >= helloLen { + h := &SniffHeader{} + err := ReadClientHello(hello[:helloLen], h) + if err == nil { + return h, nil + } + // The full handshake is present: shortages inside it are malformed, + // unlike QUIC's potentially sparse input to ReadClientHello. + if err == common.ErrNoClue || err == protocol.ErrProtoNeedMoreData { + err = errNotClientHello + } + return nil, err + } + } + b = b[5+recordLen:] + if len(b) == 0 { + return nil, protocol.ErrProtoNeedMoreData + } } - return nil, err } diff --git a/common/protocol/tls/testdata/clienthello-no-sni.bin b/common/protocol/tls/testdata/clienthello-no-sni.bin new file mode 100644 index 000000000..4b18a7f2e Binary files /dev/null and b/common/protocol/tls/testdata/clienthello-no-sni.bin differ diff --git a/common/protocol/tls/testdata/clienthello.bin b/common/protocol/tls/testdata/clienthello.bin new file mode 100644 index 000000000..f956df1b2 Binary files /dev/null and b/common/protocol/tls/testdata/clienthello.bin differ diff --git a/common/session/session.go b/common/session/session.go index 8d47bd77b..3ff43617d 100644 --- a/common/session/session.go +++ b/common/session/session.go @@ -4,6 +4,7 @@ package session // import "github.com/xtls/xray-core/common/session" import ( "context" "math/rand" + "time" c "github.com/xtls/xray-core/common/ctx" "github.com/xtls/xray-core/common/errors" @@ -77,6 +78,7 @@ type Outbound struct { // SniffingRequest controls the behavior of content sniffing. They are from inbound config. Read-only type SniffingRequest struct { + TLSSniffingTimeout time.Duration ExcludeForDomain geodata.DomainMatcher ExcludeForIP geodata.IPMatcher OverrideDestinationForProtocol []string diff --git a/infra/conf/tls_sniff_test.go b/infra/conf/tls_sniff_test.go new file mode 100644 index 000000000..8f7de9e3f --- /dev/null +++ b/infra/conf/tls_sniff_test.go @@ -0,0 +1,34 @@ +package conf_test + +import ( + "encoding/json" + "testing" + + "github.com/xtls/xray-core/infra/conf" +) + +func TestTLSSniffingJSON(t *testing.T) { + for _, tc := range []struct { + input string + valid bool + }{ + {`{"enabled":true,"destOverride":["https"],"tlsSniffingTimeout":3000}`, true}, + {`{"enabled":true,"destOverride":["tls"],"tlsSniffingTimeout":-1}`, false}, + {`{"enabled":true,"destOverride":["tls"],"tlsSniffingTimeout":60001}`, false}, + {`{"enabled":true,"destOverride":["http"],"tlsSniffingTimeout":3000}`, false}, + {`{"enabled":true,"metadataOnly":true,"destOverride":["tls"],"tlsSniffingTimeout":3000}`, false}, + } { + var config conf.SniffingConfig + err := json.Unmarshal([]byte(tc.input), &config) + if err == nil { + built, buildErr := config.Build() + err = buildErr + if err == nil && built.TlsSniffingTimeout != 3000 { + t.Fatalf("timeout lost: %v", built) + } + } + if (err == nil) != tc.valid { + t.Fatalf("%s: %v", tc.input, err) + } + } +} diff --git a/infra/conf/xray.go b/infra/conf/xray.go index 6bf6ba444..d3aeb8445 100644 --- a/infra/conf/xray.go +++ b/infra/conf/xray.go @@ -61,12 +61,13 @@ var ( ) type SniffingConfig struct { - Enabled bool `json:"enabled"` - DestOverride StringList `json:"destOverride"` - DomainsExcluded StringList `json:"domainsExcluded"` - IPsExcluded StringList `json:"ipsExcluded"` - MetadataOnly bool `json:"metadataOnly"` - RouteOnly bool `json:"routeOnly"` + TLSSniffingTimeout uint32 `json:"tlsSniffingTimeout"` + Enabled bool `json:"enabled"` + DestOverride StringList `json:"destOverride"` + DomainsExcluded StringList `json:"domainsExcluded"` + IPsExcluded StringList `json:"ipsExcluded"` + MetadataOnly bool `json:"metadataOnly"` + RouteOnly bool `json:"routeOnly"` } // Build implements Buildable. @@ -97,14 +98,16 @@ func (c *SniffingConfig) Build() (*proxyman.SniffingConfig, error) { return nil, err } - return &proxyman.SniffingConfig{ + config := &proxyman.SniffingConfig{ Enabled: c.Enabled, DestinationOverride: protocols, DomainsExcluded: domains, IpsExcluded: ips, MetadataOnly: c.MetadataOnly, RouteOnly: c.RouteOnly, - }, nil + TlsSniffingTimeout: c.TLSSniffingTimeout, + } + return config, config.Validate() } type MuxConfig struct {