diff --git a/adapter/outbound/base.go b/adapter/outbound/base.go
index cb959ef8..3e0eeadc 100644
--- a/adapter/outbound/base.go
+++ b/adapter/outbound/base.go
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"github.com/gofrs/uuid"
 	"net"
+	"strings"
 
 	"github.com/Dreamacro/clash/component/dialer"
 	C "github.com/Dreamacro/clash/constant"
@@ -128,12 +129,12 @@ func NewBase(opt BaseOption) *Base {
 
 type conn struct {
 	net.Conn
-	chain             C.Chain
-	lastAdapterRemote string
+	chain                   C.Chain
+	actualRemoteDestination string
 }
 
 func (c *conn) RemoteDestination() string {
-	return c.lastAdapterRemote
+	return c.actualRemoteDestination
 }
 
 // Chains implements C.Connection
@@ -147,17 +148,24 @@ func (c *conn) AppendToChains(a C.ProxyAdapter) {
 }
 
 func NewConn(c net.Conn, a C.ProxyAdapter) C.Conn {
-	return &conn{c, []string{a.Name()}, a.Addr()}
+	var remoteDestination string
+	if tcpAddr, ok := c.RemoteAddr().(*net.TCPAddr); ok {
+		remoteDestination = tcpAddr.IP.String()
+	} else {
+		remoteDestination = parseRemoteDestination(a.Addr())
+	}
+
+	return &conn{c, []string{a.Name()}, remoteDestination}
 }
 
 type packetConn struct {
 	net.PacketConn
-	chain             C.Chain
-	lastAdapterRemote string
+	chain                   C.Chain
+	actualRemoteDestination string
 }
 
 func (c *packetConn) RemoteDestination() string {
-	return c.lastAdapterRemote
+	return c.actualRemoteDestination
 }
 
 // Chains implements C.Connection
@@ -171,5 +179,17 @@ func (c *packetConn) AppendToChains(a C.ProxyAdapter) {
 }
 
 func newPacketConn(pc net.PacketConn, a C.ProxyAdapter) C.PacketConn {
-	return &packetConn{pc, []string{a.Name()}, a.Addr()}
+	return &packetConn{pc, []string{a.Name()}, parseRemoteDestination(a.Addr())}
+}
+
+func parseRemoteDestination(addr string) string {
+	if dst, _, err := net.SplitHostPort(addr); err == nil {
+		return dst
+	} else {
+		if addrError, ok := err.(*net.AddrError); ok && strings.Contains(addrError.Err, "missing port") {
+			return dst
+		} else {
+			return ""
+		}
+	}
 }
diff --git a/tunnel/statistic/tracker.go b/tunnel/statistic/tracker.go
index 1f5f1f9c..d176bae7 100644
--- a/tunnel/statistic/tracker.go
+++ b/tunnel/statistic/tracker.go
@@ -59,6 +59,7 @@ func (tt *tcpTracker) Close() error {
 
 func NewTCPTracker(conn C.Conn, manager *Manager, metadata *C.Metadata, rule C.Rule) *tcpTracker {
 	uuid, _ := uuid.NewV4()
+	metadata.RemoteDst = conn.RemoteDestination()
 
 	t := &tcpTracker{
 		Conn:    conn,
@@ -116,6 +117,7 @@ func (ut *udpTracker) Close() error {
 
 func NewUDPTracker(conn C.PacketConn, manager *Manager, metadata *C.Metadata, rule C.Rule) *udpTracker {
 	uuid, _ := uuid.NewV4()
+	metadata.RemoteDst = conn.RemoteDestination()
 
 	ut := &udpTracker{
 		PacketConn: conn,
diff --git a/tunnel/tunnel.go b/tunnel/tunnel.go
index 22f2c507..1b7e7514 100644
--- a/tunnel/tunnel.go
+++ b/tunnel/tunnel.go
@@ -9,7 +9,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -288,15 +287,6 @@ func handleUDPConn(packet *inbound.PacketAdapter) {
 		}
 		pCtx.InjectPacketConn(rawPc)
 
-		addr := rawPc.RemoteDestination()
-		if dst, _, err := net.SplitHostPort(addr); err == nil {
-			metadata.RemoteDst = dst
-		} else {
-			if addrError, ok := err.(*net.AddrError); ok && strings.Contains(addrError.Err, "missing port") {
-				metadata.RemoteDst = addr
-			}
-		}
-
 		pc := statistic.NewUDPTracker(rawPc, statistic.DefaultManager, metadata, rule)
 
 		switch true {
@@ -359,10 +349,6 @@ func handleTCPConn(connCtx C.ConnContext) {
 		return
 	}
 
-	if tcpAddr, ok := remoteConn.RemoteAddr().(*net.TCPAddr); ok {
-		metadata.RemoteDst = tcpAddr.IP.String()
-	}
-
 	remoteConn = statistic.NewTCPTracker(remoteConn, statistic.DefaultManager, metadata, rule)
 	defer func(remoteConn C.Conn) {
 		_ = remoteConn.Close()