From bd61487dd9f87754f962fcc4b189978aa2e50464 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 17 May 2024 17:56:12 +0200 Subject: [PATCH 01/15] Add peer not found error message --- .vscode/launch.json | 2 +- cmd/psweb/ln/lnd.go | 5 +++++ cmd/psweb/main.go | 27 +++++++++++++++------------ cmd/psweb/static/styles.css | 6 +++++- cmd/psweb/templates/reusable.gohtml | 4 ++-- 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ab05dd0..06ff138 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,7 +13,7 @@ "program": "${workspaceFolder}/cmd/psweb/", "showLog": false, "envFile": "${workspaceFolder}/.env", - "args": ["-datadir", "/home/vlad/.peerswap2"] + //"args": ["-datadir", "/home/vlad/.peerswap2"] } ] } diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index eadfebe..790a106 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -969,6 +969,11 @@ func ListPeers(client lnrpc.LightningClient, peerId string, excludeIds *[]string } } + if peerId != "" && len(peers) == 0 { + // none found + return nil, errors.New("Peer " + peerId + " not found") + } + list := peerswaprpc.ListPeersResponse{ Peers: peers, } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index fe46b78..bac2179 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -289,6 +289,10 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { _, ok = r.URL.Query()["showall"] if ok { otherPeersTable = convertOtherPeersToHTMLTable(otherPeers) + if otherPeersTable == "" && popupMessage == "" { + popupMessage = "🥳 Congratulations, all your peers use PeerSwap!" + listSwaps = convertSwapsToHTMLTable(swaps, nodeId, state, role) + } } else { listSwaps = convertSwapsToHTMLTable(swaps, nodeId, state, role) } @@ -400,8 +404,7 @@ func peerHandler(w http.ResponseWriter, r *http.Request) { // Search amoung all Lighting peers res, err := ln.ListPeers(cl, id, nil) if err != nil { - log.Printf("unable to find peer by id: %v", id) - redirectWithError(w, r, "/?", errors.New("unable to find peer by id")) + redirectWithError(w, r, "/?", err) return } peer = res.GetPeers()[0] @@ -427,11 +430,11 @@ func peerHandler(w http.ResponseWriter, r *http.Request) { sumRemote += ch.GetRemoteBalance() } - //check for error message to display - message := "" + //check for error errorMessage to display + errorMessage := "" keys, ok = r.URL.Query()["err"] if ok && len(keys[0]) > 0 { - message = keys[0] + errorMessage = keys[0] } // get routing stats @@ -465,7 +468,7 @@ func peerHandler(w http.ResponseWriter, r *http.Request) { } data := Page{ - ErrorMessage: message, + ErrorMessage: errorMessage, PopUpMessage: "", BtcFeeRate: mempoolFeeRate, MempoolFeeRate: feeRate, @@ -826,18 +829,18 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { switch action { case "keySend": + dest := r.FormValue("nodeId") + message := r.FormValue("keysendMessage") + amount, err := strconv.ParseInt(r.FormValue("keysendAmount"), 10, 64) if err != nil { - redirectWithError(w, r, "/liquid?", err) + redirectWithError(w, r, "/peer?id="+dest+"&", err) return } - dest := r.FormValue("nodeId") - message := r.FormValue("keysendMessage") - err = ln.SendKeysendMessage(dest, amount, message) if err != nil { - redirectWithError(w, r, "/liquid?", err) + redirectWithError(w, r, "/peer?id="+dest+"&", err) return } @@ -2090,7 +2093,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { peerTable += "" peerTable += "🙁 " - peerTable += "" + getNodeAlias(peer.NodeId) + peerTable += "" + getNodeAlias(peer.NodeId) peerTable += "" peerTable += "" diff --git a/cmd/psweb/static/styles.css b/cmd/psweb/static/styles.css index 2d776af..caf67d3 100644 --- a/cmd/psweb/static/styles.css +++ b/cmd/psweb/static/styles.css @@ -85,16 +85,20 @@ pre { display: flex; justify-content: space-between; align-items: center; - padding: 5px; + padding: 0px; } .left-logo { order: 1; /* This will ensure it's placed far left */ + padding-left: 5px; + padding-top: 3px; } .right-logos { display: flex; order: 2; /* This will ensure it's placed far right */ + padding-right: 5px; + padding-top: 4px; } .logo { diff --git a/cmd/psweb/templates/reusable.gohtml b/cmd/psweb/templates/reusable.gohtml index 072a673..c1fd3f4 100644 --- a/cmd/psweb/templates/reusable.gohtml +++ b/cmd/psweb/templates/reusable.gohtml @@ -18,10 +18,10 @@
-

{{.MempoolFeeRate}} sat/vB

+

{{.MempoolFeeRate}} sat/vB

From d32fbaff52185039a716b3e797572c4f79d7e573 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 17 May 2024 22:34:13 +0200 Subject: [PATCH 02/15] Optimize forwards caching --- cmd/psweb/ln/cln.go | 4 +- cmd/psweb/ln/lnd.go | 216 +++++++++++++++++++++++++------- cmd/psweb/main.go | 15 ++- cmd/psweb/templates/peer.gohtml | 2 +- 4 files changed, 185 insertions(+), 52 deletions(-) diff --git a/cmd/psweb/ln/cln.go b/cmd/psweb/ln/cln.go index 267a389..46d9c81 100644 --- a/cmd/psweb/ln/cln.go +++ b/cmd/psweb/ln/cln.go @@ -369,8 +369,8 @@ var forwards struct { Forwards []Forwarding `json:"forwards"` } -// fetch routing statistics from cln -func FetchForwardingStats() { +// cache routing history per channel from cln +func CacheForwards() { // refresh history client, clean, err := GetClient() if err != nil { diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index 790a106..ea6a252 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -41,8 +41,18 @@ import ( const Implementation = "LND" var ( - LndVerson = float64(0) // must be 0.18+ for RBF ability - forwardingEvents []*lnrpc.ForwardingEvent + LndVerson = float64(0) // must be 0.18+ for RBF ability + + // arrays mapped per channel + forwardsIn = make(map[uint64][]*lnrpc.ForwardingEvent) + forwardsOut = make(map[uint64][]*lnrpc.ForwardingEvent) + paymentHtlcs = make(map[uint64][]*lnrpc.HTLCAttempt) + invoiceHtlcs = make(map[uint64][]*lnrpc.InvoiceHTLC) + + lastforwardCreationTs uint64 + lastPaymentCreationTs int64 + lastInvoiceCreationTs int64 + // default lock id used by LND internalLockId = []byte{ 0xed, 0xe1, 0x9a, 0x92, 0xed, 0x32, 0x1a, 0x47, @@ -651,8 +661,8 @@ func GetMyAlias() string { return myNodeAlias } -// fetch all routing statistics from lnd -func FetchForwardingStats() { +// cache routing history per channel from lnd +func CacheForwards() { // refresh history client, cleanup, err := GetClient() if err != nil { @@ -663,9 +673,9 @@ func FetchForwardingStats() { // only go back 6 months start := uint64(time.Now().AddDate(0, -6, 0).Unix()) - if len(forwardingEvents) > 0 { + if lastforwardCreationTs > 0 { // continue from the last timestamp in seconds - start = forwardingEvents[len(forwardingEvents)-1].TimestampNs/1_000_000_000 + 1 + start = lastforwardCreationTs + 1 } offset := uint32(0) @@ -680,14 +690,137 @@ func FetchForwardingStats() { return } - forwardingEvents = append(forwardingEvents, res.ForwardingEvents...) - if len(res.ForwardingEvents) < 50000 { + // sort by in and out channels + for _, event := range res.ForwardingEvents { + forwardsIn[event.ChanIdIn] = append(forwardsIn[event.ChanIdIn], event) + forwardsOut[event.ChanIdOut] = append(forwardsOut[event.ChanIdOut], event) + } + + n := len(res.ForwardingEvents) + if n > 0 { + // store the last timestamp + lastforwardCreationTs = res.ForwardingEvents[n-1].TimestampNs / 1_000_000_000 + } + if n < 50000 { + // all events retrieved + break + } + + // next pull start from the next index + offset = res.LastOffsetIndex + } +} + +// cache all payments from lnd +func CachePayments() { + // refresh history + client, cleanup, err := GetClient() + if err != nil { + return + } + defer cleanup() + + // only go back 6 months + start := uint64(time.Now().AddDate(0, -6, 0).Unix()) + + if lastPaymentCreationTs > 0 { + // continue from the last timestamp in seconds + start = uint64(lastPaymentCreationTs + 1) + } + + offset := uint64(0) + for { + res, err := client.ListPayments(context.Background(), &lnrpc.ListPaymentsRequest{ + CreationDateStart: start, + IncludeIncomplete: false, + Reversed: false, + IndexOffset: offset, + MaxPayments: 50000, + }) + if err != nil { + return + } + + // only append settled ones + for _, payment := range res.Payments { + if payment.Status == lnrpc.Payment_SUCCEEDED { + for _, htlc := range payment.Htlcs { + if htlc.Status == lnrpc.HTLCAttempt_SUCCEEDED { + // get channel from the first hop + chanId := htlc.Route.Hops[0].ChanId + paymentHtlcs[chanId] = append(paymentHtlcs[chanId], htlc) + } + } + } + } + + n := len(res.Payments) + if n > 0 { + // store the last timestamp + lastPaymentCreationTs = res.Payments[n-1].CreationTimeNs / 1_000_000_000 + } + if n < 50000 { // all events retrieved break } // next pull start from the next index - offset = res.LastOffsetIndex + 1 + offset = res.LastIndexOffset + } +} + +// cache all invoices from lnd +func CacheInvoices() { + // refresh history + client, cleanup, err := GetClient() + if err != nil { + return + } + defer cleanup() + + // only go back 6 months + start := uint64(time.Now().AddDate(0, -6, 0).Unix()) + + if lastInvoiceCreationTs > 0 { + // continue from the last timestamp in seconds + start = uint64(lastInvoiceCreationTs + 1) + } + + offset := uint64(0) + for { + res, err := client.ListInvoices(context.Background(), &lnrpc.ListInvoiceRequest{ + CreationDateStart: start, + Reversed: false, + IndexOffset: offset, + NumMaxInvoices: 50000, + }) + if err != nil { + return + } + + // only append settled htlcs + for _, invoice := range res.Invoices { + if invoice.State == lnrpc.Invoice_SETTLED { + for _, htlc := range invoice.Htlcs { + if htlc.State == lnrpc.InvoiceHTLCState_SETTLED { + invoiceHtlcs[htlc.ChanId] = append(invoiceHtlcs[htlc.ChanId], htlc) + } + } + } + } + + n := len(res.Invoices) + if n > 0 { + // store the last timestamp + lastInvoiceCreationTs = res.Invoices[n-1].CreationDate + } + if n < 50000 { + // all invoices retrieved + break + } + + // next pull start from the next index + offset = res.LastIndexOffset } } @@ -709,32 +842,31 @@ func GetForwardingStats(channelId uint64) *ForwardingStats { timestamp30d := uint64(now.AddDate(0, 0, -30).Unix()) * 1_000_000_000 timestamp6m := uint64(now.AddDate(0, -6, 0).Unix()) * 1_000_000_000 - for _, e := range forwardingEvents { - if e.ChanIdOut == channelId { - if e.TimestampNs > timestamp6m { - result.AmountOut6m += e.AmtOut - feeMsat6m += e.FeeMsat - if e.TimestampNs > timestamp30d { - result.AmountOut30d += e.AmtOut - feeMsat30d += e.FeeMsat - if e.TimestampNs > timestamp7d { - result.AmountOut7d += e.AmtOut - feeMsat7d += e.FeeMsat - } + for _, e := range forwardsOut[channelId] { + if e.TimestampNs > timestamp6m { + result.AmountOut6m += e.AmtOut + feeMsat6m += e.FeeMsat + if e.TimestampNs > timestamp30d { + result.AmountOut30d += e.AmtOut + feeMsat30d += e.FeeMsat + if e.TimestampNs > timestamp7d { + result.AmountOut7d += e.AmtOut + feeMsat7d += e.FeeMsat } } } - if e.ChanIdIn == channelId { - if e.TimestampNs > timestamp6m { - result.AmountIn6m += e.AmtIn - assistedMsat6m += e.FeeMsat - if e.TimestampNs > timestamp30d { - result.AmountIn30d += e.AmtIn - assistedMsat30d += e.FeeMsat - if e.TimestampNs > timestamp7d { - result.AmountIn7d += e.AmtIn - assistedMsat7d += e.FeeMsat - } + } + + for _, e := range forwardsIn[channelId] { + if e.TimestampNs > timestamp6m { + result.AmountIn6m += e.AmtIn + assistedMsat6m += e.FeeMsat + if e.TimestampNs > timestamp30d { + result.AmountIn30d += e.AmtIn + assistedMsat30d += e.FeeMsat + if e.TimestampNs > timestamp7d { + result.AmountIn7d += e.AmtIn + assistedMsat7d += e.FeeMsat } } } @@ -804,18 +936,16 @@ func GetForwardingStatsSinceTS(channelId uint64, timeStamp uint64) *ShortForward timestampNs := timeStamp * 1_000_000_000 - for _, e := range forwardingEvents { - if e.ChanIdOut == channelId { - if e.TimestampNs > timestampNs { - result.AmountOut += e.AmtOut - feeMsat += e.FeeMsat - } + for _, e := range forwardsOut[channelId] { + if e.TimestampNs > timestampNs { + result.AmountOut += e.AmtOut + feeMsat += e.FeeMsat } - if e.ChanIdIn == channelId { - if e.TimestampNs > timestampNs { - result.AmountIn += e.AmtIn - assistedMsat += e.FeeMsat - } + } + for _, e := range forwardsIn[channelId] { + if e.TimestampNs > timestampNs { + result.AmountIn += e.AmtIn + assistedMsat += e.FeeMsat } } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index bac2179..8b4bab1 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -1388,8 +1388,10 @@ func onTimer() { // Check if pegin can be claimed go checkPegin() - // pre-cache routing statistics - go ln.FetchForwardingStats() + // cache flow history + go ln.CacheForwards() + go ln.CachePayments() + go ln.CacheInvoices() // check for updates t := internet.GetLatestTag() @@ -1749,16 +1751,17 @@ func bumpfeeHandler(w http.ResponseWriter, r *http.Request) { return } - // to speed things up, also broadcast it to mempool.space - internet.SendRawTransaction(res.RawHex) - if ln.CanRBF() { + // to speed things up, also broadcast it to mempool.space + internet.SendRawTransaction(res.RawHex) + log.Println("RBF TxId:", res.TxId) config.Config.PeginReplacedTxId = config.Config.PeginTxId config.Config.PeginAmount = res.AmountSat config.Config.PeginTxId = res.TxId } else { - log.Println("CPFP successful") + // txid not available, let's hope LND broadcasted it fine + log.Println("CPFP initiated") } // save the new rate, so the next bump cannot be lower diff --git a/cmd/psweb/templates/peer.gohtml b/cmd/psweb/templates/peer.gohtml index d2f3f45..6eb708e 100644 --- a/cmd/psweb/templates/peer.gohtml +++ b/cmd/psweb/templates/peer.gohtml @@ -221,7 +221,7 @@ Sincerely, {{end}} - + From 8c9a632e029850ccf0d9a13a88c9a36ee72a8a4d Mon Sep 17 00:00:00 2001 From: Impa10r Date: Fri, 17 May 2024 23:32:24 +0200 Subject: [PATCH 03/15] Correct flows --- CHANGELOG.md | 5 +++ cmd/psweb/ln/common.go | 9 +++-- cmd/psweb/ln/lnd.go | 35 +++++++++++++---- cmd/psweb/main.go | 86 +++++++++++++++++++++--------------------- 4 files changed, 81 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1212863..dd1910a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Versions +## 1.4.1 + +- Account to paid and received invoices +- Show circular rebalancing costs + ## 1.4.0 - Enable viewing non-PeerSwap channels diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index f3c7e8e..8e5e536 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -44,11 +44,14 @@ type ForwardingStats struct { AssistedPPM6m uint64 } -type ShortForwardingStats struct { - AmountOut uint64 - AmountIn uint64 +type ChannelStats struct { + RoutedOut uint64 + RoutedIn uint64 FeeSat uint64 AssistedFeeSat uint64 + PaidOut uint64 + InvoicedIn uint64 + RebalanceCost uint64 } type ChanneInfo struct { diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index ea6a252..5d16b2b 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -925,32 +925,53 @@ func GetChannelInfo(client lnrpc.LightningClient, channelId uint64, nodeId strin return info } -// forwarding stats for a channel since timestamp -func GetForwardingStatsSinceTS(channelId uint64, timeStamp uint64) *ShortForwardingStats { +// flow stats for a channel since timestamp +func GetChannelStats(channelId uint64, timeStamp uint64) *ChannelStats { var ( - result ShortForwardingStats - feeMsat uint64 - assistedMsat uint64 + result ChannelStats + feeMsat uint64 + assistedMsat uint64 + paidOutMsat int64 + invoicedMsat uint64 + rebalCostMsat int64 ) timestampNs := timeStamp * 1_000_000_000 for _, e := range forwardsOut[channelId] { if e.TimestampNs > timestampNs { - result.AmountOut += e.AmtOut + result.RoutedOut += e.AmtOut feeMsat += e.FeeMsat } } for _, e := range forwardsIn[channelId] { if e.TimestampNs > timestampNs { - result.AmountIn += e.AmtIn + result.RoutedIn += e.AmtIn assistedMsat += e.FeeMsat } } + for _, e := range invoiceHtlcs[channelId] { + if uint64(e.ResolveTime) > timestampNs { + invoicedMsat += e.AmtMsat + } + } + for _, e := range paymentHtlcs[channelId] { + if uint64(e.ResolveTimeNs) > timestampNs { + paidOutMsat += e.Route.TotalAmtMsat + // identify circular rebalancing + n := len(e.Route.Hops) + if n > 0 && e.Route.Hops[n-1].ChanId == channelId { + rebalCostMsat += e.Route.TotalFeesMsat + } + } + } result.FeeSat = feeMsat / 1000 result.AssistedFeeSat = assistedMsat / 1000 + result.InvoicedIn = invoicedMsat / 1000 + result.PaidOut = uint64(paidOutMsat / 1000) + result.RebalanceCost = uint64(rebalCostMsat / 1000) return &result } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 8b4bab1..201b904 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -33,7 +33,7 @@ import ( const ( // App version tag - version = "v1.4.0" + version = "v1.4.1" // Liquid balance to reserve in auto swaps // Min is 1000, but the swap will spend it all on fee @@ -1841,10 +1841,10 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers for _, peer := range peers { var totalLocal float64 var totalCapacity float64 - var totalOutflows uint64 - var totalInflows uint64 + var totalForwardsOut uint64 + var totalForwardsIn uint64 var totalFees uint64 - var totalAssistedFees uint64 + var totalRebalCost uint64 channelsTable := "
FlowForwards Out Revenue In
" @@ -1883,13 +1883,15 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers tooltip = " since the last swap " + timePassedAgo(time.Unix(lastSwapTimestamp, 0).UTC()) } - stats := ln.GetForwardingStatsSinceTS(channel.ChannelId, uint64(lastSwapTimestamp)) + stats := ln.GetChannelStats(channel.ChannelId, uint64(lastSwapTimestamp)) totalFees += stats.FeeSat - totalAssistedFees += stats.AssistedFeeSat - totalOutflows += stats.AmountOut - totalInflows += stats.AmountIn + outflows := stats.RoutedOut + stats.PaidOut + inflows := stats.RoutedIn + stats.InvoicedIn + totalForwardsOut += stats.RoutedOut + totalForwardsIn += stats.RoutedIn + totalRebalCost += stats.RebalanceCost - netFlow := float64(int64(stats.AmountIn) - int64(stats.AmountOut)) + netFlow := float64(int64(inflows) - int64(outflows)) bluePct := int(local * 100 / capacity) greenPct := int(0) @@ -1897,7 +1899,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers previousBlue := bluePct previousRed := redPct - tooltip = fmt.Sprintf("%d", bluePct) + "% local, Inflows: " + toMil(stats.AmountIn) + ", outflows: " + toMil(stats.AmountOut) + tooltip + tooltip = fmt.Sprintf("%d", bluePct) + "% local, Inflows: " + toMil(inflows) + ", outflows: " + toMil(outflows) + tooltip if netFlow > 0 { greenPct = int(local * 100 / capacity) @@ -1946,20 +1948,17 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers peerTable += "" + getNodeAlias(peer.NodeId) peerTable += "" - peerTable += "
" + peerTable += "" - ppm := uint64(0) - if totalOutflows > 0 { - ppm = totalFees * 1_000_000 / totalOutflows + ppmRevenue := uint64(0) + ppmCost := uint64(0) + if totalForwardsOut > 0 { + ppmRevenue = totalFees * 1_000_000 / totalForwardsOut + ppmCost = totalRebalCost * 1_000_000 / totalForwardsOut } - peerTable += "" + formatWithThousandSeparators(totalFees) + " / " - - ppm = 0 - if totalInflows > 0 { - ppm = totalAssistedFees * 1_000_000 / totalInflows - } - peerTable += "" + formatWithThousandSeparators(totalAssistedFees) + "" + peerTable += "" + formatWithThousandSeparators(totalFees) + " / " + peerTable += "" + formatWithThousandSeparators(totalRebalCost) + "" peerTable += "" if stringIsInSlice("lbtc", peer.SupportedAssets) { @@ -2010,10 +2009,10 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { for _, peer := range peers { var totalLocal float64 var totalCapacity float64 - var totalOutflows uint64 - var totalInflows uint64 + var totalForwardsOut uint64 + var totalForwardsIn uint64 var totalFees uint64 - var totalAssistedFees uint64 + var totalRebalCost uint64 channelsTable := "" @@ -2045,13 +2044,15 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { totalCapacity += capacity tooltip := " in the last 6 months" - stats := ln.GetForwardingStatsSinceTS(channel.ChannelId, uint64(lastSwapTimestamp)) + stats := ln.GetChannelStats(channel.ChannelId, uint64(lastSwapTimestamp)) totalFees += stats.FeeSat - totalAssistedFees += stats.AssistedFeeSat - totalOutflows += stats.AmountOut - totalInflows += stats.AmountIn + outflows := stats.RoutedOut + stats.PaidOut + inflows := stats.RoutedIn + stats.InvoicedIn + totalForwardsOut += stats.RoutedOut + totalForwardsIn += stats.RoutedIn + totalRebalCost += stats.RebalanceCost - netFlow := float64(int64(stats.AmountIn) - int64(stats.AmountOut)) + netFlow := float64(int64(inflows) - int64(outflows)) bluePct := int(local * 100 / capacity) greenPct := int(0) @@ -2059,7 +2060,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { previousBlue := bluePct previousRed := redPct - tooltip = fmt.Sprintf("%d", bluePct) + "% local, Inflows: " + toMil(stats.AmountIn) + ", outflows: " + toMil(stats.AmountOut) + tooltip + tooltip = fmt.Sprintf("%d", bluePct) + "% local, Inflows: " + toMil(inflows) + ", outflows: " + toMil(outflows) + tooltip if netFlow > 0 { greenPct = int(local * 100 / capacity) @@ -2099,19 +2100,16 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { peerTable += "" + getNodeAlias(peer.NodeId) peerTable += "" - peerTable += "
" - - ppm := uint64(0) - if totalOutflows > 0 { - ppm = totalFees * 1_000_000 / totalOutflows - } - peerTable += "" + formatWithThousandSeparators(totalFees) + " / " + peerTable += "" - ppm = 0 - if totalInflows > 0 { - ppm = totalAssistedFees * 1_000_000 / totalInflows + ppmRevenue := uint64(0) + ppmCost := uint64(0) + if totalForwardsOut > 0 { + ppmRevenue = totalFees * 1_000_000 / totalForwardsOut + ppmCost = totalRebalCost * 1_000_000 / totalForwardsOut } - peerTable += "" + formatWithThousandSeparators(totalAssistedFees) + "" + peerTable += "" + formatWithThousandSeparators(totalFees) + " / " + peerTable += "" + formatWithThousandSeparators(totalRebalCost) + "" peerTable += "" peerTable += "Invite " peerTable += "
" @@ -2410,11 +2408,11 @@ func findSwapInCandidate(candidate *SwapParams) error { lastSwapTimestamp = swapTimestamps[channel.ChannelId] } - stats := ln.GetForwardingStatsSinceTS(channel.ChannelId, uint64(lastSwapTimestamp)) + stats := ln.GetChannelStats(channel.ChannelId, uint64(lastSwapTimestamp)) ppm := uint64(0) - if stats.AmountOut > 0 { - ppm = stats.FeeSat * 1_000_000 / stats.AmountOut + if stats.RoutedOut > 0 { + ppm = stats.FeeSat * 1_000_000 / stats.RoutedOut } // aim to maximize accumulated PPM From f475920eb31bb2b8a891e73e82eba66d5cb97066 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 00:17:34 +0200 Subject: [PATCH 04/15] Ignore swap payments --- cmd/psweb/main.go | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 201b904..c305fe9 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -1832,7 +1832,11 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers for _, swap := range swaps { if simplifySwapState(swap.State) == "success" && swapTimestamps[swap.LndChanId] < swap.CreatedAt { - swapTimestamps[swap.LndChanId] = swap.CreatedAt + extraSeconds := int64(300) // 5 mins to ignore lighting payments related to swap + if swap.Asset == "btc" { + extraSeconds = 1200 // 20 mins for BTC swaps + } + swapTimestamps[swap.LndChanId] = swap.CreatedAt + extraSeconds } } @@ -1874,13 +1878,13 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers capacity := float64(channel.LocalBalance + channel.RemoteBalance) totalLocal += local totalCapacity += capacity - tooltip := " in the last 6 months" + tooltip := "In the last 6 months" // timestamp of the last swap or 6 months horizon lastSwapTimestamp := time.Now().AddDate(0, -6, 0).Unix() if swapTimestamps[channel.ChannelId] > lastSwapTimestamp { lastSwapTimestamp = swapTimestamps[channel.ChannelId] - tooltip = " since the last swap " + timePassedAgo(time.Unix(lastSwapTimestamp, 0).UTC()) + tooltip = "Since the last swap " + timePassedAgo(time.Unix(lastSwapTimestamp, 0).UTC()) } stats := ln.GetChannelStats(channel.ChannelId, uint64(lastSwapTimestamp)) @@ -1899,19 +1903,32 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers previousBlue := bluePct previousRed := redPct - tooltip = fmt.Sprintf("%d", bluePct) + "% local, Inflows: " + toMil(inflows) + ", outflows: " + toMil(outflows) + tooltip + tooltip = fmt.Sprintf("%d", bluePct) + "% local balance\n" + tooltip + ":" + if stats.RoutedIn > 0 { + tooltip += "\nRouted in: +" + formatWithThousandSeparators(stats.RoutedIn) + } + if stats.InvoicedIn > 0 { + tooltip += "\nInvoiced in: +" + formatWithThousandSeparators(stats.InvoicedIn) + } + if stats.RoutedOut > 0 { + tooltip += "\nRouted out: -" + formatWithThousandSeparators(stats.RoutedOut) + } + if stats.PaidOut > 0 { + tooltip += "\nPaid out: -" + formatWithThousandSeparators(stats.PaidOut) + } if netFlow > 0 { greenPct = int(local * 100 / capacity) bluePct = int((local - netFlow) * 100 / capacity) previousBlue = greenPct - + tooltip += "\nNet flow: +" + formatWithThousandSeparators(uint64(netFlow)) } if netFlow < 0 { bluePct = int(local * 100 / capacity) redPct = int((local - netFlow) * 100 / capacity) previousRed = bluePct + tooltip += "\nNet flow: -" + formatWithThousandSeparators(uint64(-netFlow)) } currentProgress := fmt.Sprintf("%d%% 100%%, %d%% 100%%, %d%% 100%%, 100%% 100%%", bluePct, redPct, greenPct) From 18466c085650d77915b91d8c4e10aaa0d3b0f334 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 00:37:19 +0200 Subject: [PATCH 05/15] Show costs --- cmd/psweb/ln/common.go | 2 +- cmd/psweb/ln/lnd.go | 20 +++++-------- cmd/psweb/main.go | 66 ++++++++++++++++++++++++++++++------------ 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index 8e5e536..91c2e20 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -51,7 +51,7 @@ type ChannelStats struct { AssistedFeeSat uint64 PaidOut uint64 InvoicedIn uint64 - RebalanceCost uint64 + PaidCost uint64 } type ChanneInfo struct { diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index 5d16b2b..bcaffff 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -929,12 +929,12 @@ func GetChannelInfo(client lnrpc.LightningClient, channelId uint64, nodeId strin func GetChannelStats(channelId uint64, timeStamp uint64) *ChannelStats { var ( - result ChannelStats - feeMsat uint64 - assistedMsat uint64 - paidOutMsat int64 - invoicedMsat uint64 - rebalCostMsat int64 + result ChannelStats + feeMsat uint64 + assistedMsat uint64 + paidOutMsat int64 + invoicedMsat uint64 + costMsat int64 ) timestampNs := timeStamp * 1_000_000_000 @@ -959,11 +959,7 @@ func GetChannelStats(channelId uint64, timeStamp uint64) *ChannelStats { for _, e := range paymentHtlcs[channelId] { if uint64(e.ResolveTimeNs) > timestampNs { paidOutMsat += e.Route.TotalAmtMsat - // identify circular rebalancing - n := len(e.Route.Hops) - if n > 0 && e.Route.Hops[n-1].ChanId == channelId { - rebalCostMsat += e.Route.TotalFeesMsat - } + costMsat += e.Route.TotalFeesMsat } } @@ -971,7 +967,7 @@ func GetChannelStats(channelId uint64, timeStamp uint64) *ChannelStats { result.AssistedFeeSat = assistedMsat / 1000 result.InvoicedIn = invoicedMsat / 1000 result.PaidOut = uint64(paidOutMsat / 1000) - result.RebalanceCost = uint64(rebalCostMsat / 1000) + result.PaidCost = uint64(costMsat / 1000) return &result } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index c305fe9..02d2e2b 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -1848,7 +1848,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers var totalForwardsOut uint64 var totalForwardsIn uint64 var totalFees uint64 - var totalRebalCost uint64 + var totalCost uint64 channelsTable := "" @@ -1893,7 +1893,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers inflows := stats.RoutedIn + stats.InvoicedIn totalForwardsOut += stats.RoutedOut totalForwardsIn += stats.RoutedIn - totalRebalCost += stats.RebalanceCost + totalCost += stats.PaidCost netFlow := float64(int64(inflows) - int64(outflows)) @@ -1904,33 +1904,40 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers previousRed := redPct tooltip = fmt.Sprintf("%d", bluePct) + "% local balance\n" + tooltip + ":" + flowText := "" if stats.RoutedIn > 0 { - tooltip += "\nRouted in: +" + formatWithThousandSeparators(stats.RoutedIn) + flowText += "\nRouted in: +" + formatWithThousandSeparators(stats.RoutedIn) } if stats.InvoicedIn > 0 { - tooltip += "\nInvoiced in: +" + formatWithThousandSeparators(stats.InvoicedIn) + flowText += "\nInvoiced in: +" + formatWithThousandSeparators(stats.InvoicedIn) } if stats.RoutedOut > 0 { - tooltip += "\nRouted out: -" + formatWithThousandSeparators(stats.RoutedOut) + flowText += "\nRouted out: -" + formatWithThousandSeparators(stats.RoutedOut) } if stats.PaidOut > 0 { - tooltip += "\nPaid out: -" + formatWithThousandSeparators(stats.PaidOut) + flowText += "\nPaid out: -" + formatWithThousandSeparators(stats.PaidOut) } if netFlow > 0 { greenPct = int(local * 100 / capacity) bluePct = int((local - netFlow) * 100 / capacity) previousBlue = greenPct - tooltip += "\nNet flow: +" + formatWithThousandSeparators(uint64(netFlow)) + flowText += "\nNet flow: +" + formatWithThousandSeparators(uint64(netFlow)) } if netFlow < 0 { bluePct = int(local * 100 / capacity) redPct = int((local - netFlow) * 100 / capacity) previousRed = bluePct - tooltip += "\nNet flow: -" + formatWithThousandSeparators(uint64(-netFlow)) + flowText += "\nNet flow: -" + formatWithThousandSeparators(uint64(-netFlow)) } + if flowText == "" { + flowText = "\nNo flows" + } + + tooltip += flowText + currentProgress := fmt.Sprintf("%d%% 100%%, %d%% 100%%, %d%% 100%%, 100%% 100%%", bluePct, redPct, greenPct) previousProgress := fmt.Sprintf("%d%% 100%%, %d%% 100%%, %d%% 100%%, 100%% 100%%", previousBlue, previousRed, greenPct) @@ -1971,11 +1978,11 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers ppmCost := uint64(0) if totalForwardsOut > 0 { ppmRevenue = totalFees * 1_000_000 / totalForwardsOut - ppmCost = totalRebalCost * 1_000_000 / totalForwardsOut + ppmCost = totalCost * 1_000_000 / totalForwardsOut } - peerTable += "" + formatWithThousandSeparators(totalFees) + " / " - peerTable += "" + formatWithThousandSeparators(totalRebalCost) + "" + peerTable += "" + formatWithThousandSeparators(totalFees) + " / " + peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "
" if stringIsInSlice("lbtc", peer.SupportedAssets) { @@ -2029,7 +2036,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { var totalForwardsOut uint64 var totalForwardsIn uint64 var totalFees uint64 - var totalRebalCost uint64 + var totalCost uint64 channelsTable := "" @@ -2059,7 +2066,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { capacity := float64(channel.LocalBalance + channel.RemoteBalance) totalLocal += local totalCapacity += capacity - tooltip := " in the last 6 months" + tooltip := "In the last 6 months" stats := ln.GetChannelStats(channel.ChannelId, uint64(lastSwapTimestamp)) totalFees += stats.FeeSat @@ -2067,7 +2074,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { inflows := stats.RoutedIn + stats.InvoicedIn totalForwardsOut += stats.RoutedOut totalForwardsIn += stats.RoutedIn - totalRebalCost += stats.RebalanceCost + totalCost += stats.PaidCost netFlow := float64(int64(inflows) - int64(outflows)) @@ -2077,21 +2084,42 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { previousBlue := bluePct previousRed := redPct - tooltip = fmt.Sprintf("%d", bluePct) + "% local, Inflows: " + toMil(inflows) + ", outflows: " + toMil(outflows) + tooltip + tooltip = fmt.Sprintf("%d", bluePct) + "% local balance\n" + tooltip + ":" + flowText := "" + + if stats.RoutedIn > 0 { + flowText += "\nRouted in: +" + formatWithThousandSeparators(stats.RoutedIn) + } + if stats.InvoicedIn > 0 { + flowText += "\nInvoiced in: +" + formatWithThousandSeparators(stats.InvoicedIn) + } + if stats.RoutedOut > 0 { + flowText += "\nRouted out: -" + formatWithThousandSeparators(stats.RoutedOut) + } + if stats.PaidOut > 0 { + flowText += "\nPaid out: -" + formatWithThousandSeparators(stats.PaidOut) + } if netFlow > 0 { greenPct = int(local * 100 / capacity) bluePct = int((local - netFlow) * 100 / capacity) previousBlue = greenPct - + flowText += "\nNet flow: +" + formatWithThousandSeparators(uint64(netFlow)) } if netFlow < 0 { bluePct = int(local * 100 / capacity) redPct = int((local - netFlow) * 100 / capacity) previousRed = bluePct + flowText += "\nNet flow: -" + formatWithThousandSeparators(uint64(-netFlow)) + } + + if flowText == "" { + flowText = "\nNo flows" } + tooltip += flowText + currentProgress := fmt.Sprintf("%d%% 100%%, %d%% 100%%, %d%% 100%%, 100%% 100%%", bluePct, redPct, greenPct) previousProgress := fmt.Sprintf("%d%% 100%%, %d%% 100%%, %d%% 100%%, 100%% 100%%", previousBlue, previousRed, greenPct) @@ -2123,10 +2151,10 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { ppmCost := uint64(0) if totalForwardsOut > 0 { ppmRevenue = totalFees * 1_000_000 / totalForwardsOut - ppmCost = totalRebalCost * 1_000_000 / totalForwardsOut + ppmCost = totalCost * 1_000_000 / totalForwardsOut } - peerTable += "" + formatWithThousandSeparators(totalFees) + " / " - peerTable += "" + formatWithThousandSeparators(totalRebalCost) + "" + peerTable += "" + formatWithThousandSeparators(totalFees) + " / " + peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "
" peerTable += "Invite " peerTable += "
" From dec53a371c9355a7e55db84f30fcfde8145f7c0e Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 01:00:03 +0200 Subject: [PATCH 06/15] Fix cost ppm --- cmd/psweb/main.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 02d2e2b..6d67830 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -62,8 +62,6 @@ var ( logFile *os.File latestVersion = version mempoolFeeRate = float64(0) - peersCache string - nonPeersCache string ) func main() { @@ -1847,6 +1845,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers var totalCapacity float64 var totalForwardsOut uint64 var totalForwardsIn uint64 + var totalPayments uint64 var totalFees uint64 var totalCost uint64 @@ -1894,6 +1893,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers totalForwardsOut += stats.RoutedOut totalForwardsIn += stats.RoutedIn totalCost += stats.PaidCost + totalPayments += stats.PaidOut netFlow := float64(int64(inflows) - int64(outflows)) @@ -1978,10 +1978,12 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers ppmCost := uint64(0) if totalForwardsOut > 0 { ppmRevenue = totalFees * 1_000_000 / totalForwardsOut - ppmCost = totalCost * 1_000_000 / totalForwardsOut + } + if totalPayments > 0 { + ppmCost = totalCost * 1_000_000 / totalPayments } - peerTable += "" + formatWithThousandSeparators(totalFees) + " / " + peerTable += "" + formatWithThousandSeparators(totalFees) + " / " peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "		" @@ -2035,6 +2037,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { var totalCapacity float64 var totalForwardsOut uint64 var totalForwardsIn uint64 + var totalPayments uint64 var totalFees uint64 var totalCost uint64 @@ -2074,6 +2077,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { inflows := stats.RoutedIn + stats.InvoicedIn totalForwardsOut += stats.RoutedOut totalForwardsIn += stats.RoutedIn + totalPayments += stats.PaidOut totalCost += stats.PaidCost netFlow := float64(int64(inflows) - int64(outflows)) @@ -2151,7 +2155,9 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { ppmCost := uint64(0) if totalForwardsOut > 0 { ppmRevenue = totalFees * 1_000_000 / totalForwardsOut - ppmCost = totalCost * 1_000_000 / totalForwardsOut + } + if totalPayments > 0 { + ppmCost = totalCost * 1_000_000 / totalPayments } peerTable += "" + formatWithThousandSeparators(totalFees) + " / " peerTable += "" + formatWithThousandSeparators(totalCost) + "" From b3ae5463e02bacb1fdef07c78da6f7ca8142a9ea Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 09:19:54 +0200 Subject: [PATCH 07/15] Fix Invoiced stats --- cmd/psweb/ln/lnd.go | 2 +- cmd/psweb/main.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index bcaffff..ad6cdb7 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -952,7 +952,7 @@ func GetChannelStats(channelId uint64, timeStamp uint64) *ChannelStats { } } for _, e := range invoiceHtlcs[channelId] { - if uint64(e.ResolveTime) > timestampNs { + if uint64(e.ResolveTime) > timeStamp { invoicedMsat += e.AmtMsat } } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 6d67830..c660ce0 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -1984,7 +1984,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers } peerTable += "" + formatWithThousandSeparators(totalFees) + " / " - peerTable += "" + formatWithThousandSeparators(totalCost) + "" + peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "" if stringIsInSlice("lbtc", peer.SupportedAssets) { @@ -2160,7 +2160,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { ppmCost = totalCost * 1_000_000 / totalPayments } peerTable += "" + formatWithThousandSeparators(totalFees) + " / " - peerTable += "" + formatWithThousandSeparators(totalCost) + "" + peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "" peerTable += "Invite " peerTable += "
" From 7128d5fbec57c90ef213108839c40040147e58e1 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 09:54:51 +0200 Subject: [PATCH 08/15] Cache html peer tables --- cmd/psweb/main.go | 62 +++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index c660ce0..80864f8 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -58,10 +58,12 @@ var ( //go:embed static staticFiles embed.FS //go:embed templates/*.gohtml - tplFolder embed.FS - logFile *os.File - latestVersion = version - mempoolFeeRate = float64(0) + tplFolder embed.FS + logFile *os.File + latestVersion = version + mempoolFeeRate = float64(0) + peerTableCache = "" + nonPeerTableCache = "" ) func main() { @@ -281,13 +283,22 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { role = keys[0] } + // use cache for page refreshes < 1 minute + peerTable := peerTableCache + if peerTable == "" { + peerTable = convertPeersToHTMLTable(peers, allowlistedPeers, suspiciousPeers, swaps) + } + //check whether to display non-PS channels or swaps - otherPeersTable := "" + nonPeerTable := "" listSwaps := "" _, ok = r.URL.Query()["showall"] if ok { - otherPeersTable = convertOtherPeersToHTMLTable(otherPeers) - if otherPeersTable == "" && popupMessage == "" { + nonPeerTable = nonPeerTableCache + if nonPeerTable == "" { + nonPeerTable = convertOtherPeersToHTMLTable(otherPeers) + } + if nonPeerTable == "" && popupMessage == "" { popupMessage = "🥳 Congratulations, all your peers use PeerSwap!" listSwaps = convertSwapsToHTMLTable(swaps, nodeId, state, role) } @@ -319,8 +330,8 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { MempoolFeeRate: mempoolFeeRate, ColorScheme: config.Config.ColorScheme, LiquidBalance: satAmount, - ListPeers: convertPeersToHTMLTable(peers, allowlistedPeers, suspiciousPeers, swaps), - OtherPeers: otherPeersTable, + ListPeers: peerTable, + OtherPeers: nonPeerTable, ListSwaps: listSwaps, BitcoinBalance: uint64(btcBalance), Filter: nodeId != "" || state != "" || role != "", @@ -1386,10 +1397,15 @@ func onTimer() { // Check if pegin can be claimed go checkPegin() - // cache flow history - go ln.CacheForwards() - go ln.CachePayments() - go ln.CacheInvoices() + // cache flow history and wipe html tables + go func() { + ln.CacheForwards() + ln.CachePayments() + ln.CacheInvoices() + + peerTableCache = "" + nonPeerTableCache = "" + }() // check for updates t := internet.GetLatestTag() @@ -1908,12 +1924,12 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers if stats.RoutedIn > 0 { flowText += "\nRouted in: +" + formatWithThousandSeparators(stats.RoutedIn) } - if stats.InvoicedIn > 0 { - flowText += "\nInvoiced in: +" + formatWithThousandSeparators(stats.InvoicedIn) - } if stats.RoutedOut > 0 { flowText += "\nRouted out: -" + formatWithThousandSeparators(stats.RoutedOut) } + if stats.InvoicedIn > 0 { + flowText += "\nInvoiced in: +" + formatWithThousandSeparators(stats.InvoicedIn) + } if stats.PaidOut > 0 { flowText += "\nPaid out: -" + formatWithThousandSeparators(stats.PaidOut) } @@ -1972,7 +1988,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers peerTable += "" + getNodeAlias(peer.NodeId) peerTable += "" - peerTable += "		" + peerTable += "" ppmRevenue := uint64(0) ppmCost := uint64(0) @@ -1983,7 +1999,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers ppmCost = totalCost * 1_000_000 / totalPayments } - peerTable += "" + formatWithThousandSeparators(totalFees) + " / " + peerTable += "" + formatWithThousandSeparators(totalFees) + " " peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "" @@ -2094,12 +2110,12 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { if stats.RoutedIn > 0 { flowText += "\nRouted in: +" + formatWithThousandSeparators(stats.RoutedIn) } - if stats.InvoicedIn > 0 { - flowText += "\nInvoiced in: +" + formatWithThousandSeparators(stats.InvoicedIn) - } if stats.RoutedOut > 0 { flowText += "\nRouted out: -" + formatWithThousandSeparators(stats.RoutedOut) } + if stats.InvoicedIn > 0 { + flowText += "\nInvoiced in: +" + formatWithThousandSeparators(stats.InvoicedIn) + } if stats.PaidOut > 0 { flowText += "\nPaid out: -" + formatWithThousandSeparators(stats.PaidOut) } @@ -2149,7 +2165,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { peerTable += "" + getNodeAlias(peer.NodeId) peerTable += "" - peerTable += "" + peerTable += "" ppmRevenue := uint64(0) ppmCost := uint64(0) @@ -2159,7 +2175,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { if totalPayments > 0 { ppmCost = totalCost * 1_000_000 / totalPayments } - peerTable += "" + formatWithThousandSeparators(totalFees) + " / " + peerTable += "" + formatWithThousandSeparators(totalFees) + " " peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "" peerTable += "Invite " From 9126e60e2e44989b5d169f4278691e0e3093dba1 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 09:57:35 +0200 Subject: [PATCH 09/15] Use cache --- cmd/psweb/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 80864f8..5788ee0 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -287,6 +287,7 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { peerTable := peerTableCache if peerTable == "" { peerTable = convertPeersToHTMLTable(peers, allowlistedPeers, suspiciousPeers, swaps) + peerTableCache = peerTable } //check whether to display non-PS channels or swaps @@ -294,9 +295,11 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { listSwaps := "" _, ok = r.URL.Query()["showall"] if ok { + // use cache for page refreshes < 1 minute nonPeerTable = nonPeerTableCache if nonPeerTable == "" { nonPeerTable = convertOtherPeersToHTMLTable(otherPeers) + nonPeerTableCache = nonPeerTable } if nonPeerTable == "" && popupMessage == "" { popupMessage = "🥳 Congratulations, all your peers use PeerSwap!" From 460bd2231b9c67e750f61990d648ae2884110750 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 10:10:33 +0200 Subject: [PATCH 10/15] Fix caching --- cmd/psweb/main.go | 62 ++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 5788ee0..a17dabf 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -193,22 +193,6 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { } defer cleanup() - res, err := ps.ReloadPolicyFile(client) - if err != nil { - redirectWithError(w, r, "/config?", err) - return - } - - allowlistedPeers := res.GetAllowlistedPeers() - suspiciousPeers := res.GetSuspiciousPeerList() - - res2, err := ps.ListPeers(client) - if err != nil { - redirectWithError(w, r, "/config?", err) - return - } - peers := res2.GetPeers() - res3, err := ps.ListSwaps(client) if err != nil { redirectWithError(w, r, "/config?", err) @@ -234,20 +218,6 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { btcBalance := ln.ConfirmedWalletBalance(cl) - var psIds []string - - for _, peer := range peers { - psIds = append(psIds, peer.NodeId) - } - - // Get the remaining Lightning peers - res5, err := ln.ListPeers(cl, "", &psIds) - if err != nil { - redirectWithError(w, r, "/config?", err) - return - } - otherPeers := res5.GetPeers() - //check for error message to display errorMessage := "" keys, ok := r.URL.Query()["err"] @@ -283,9 +253,27 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { role = keys[0] } + var peers []*peerswaprpc.PeerSwapPeer + // use cache for page refreshes < 1 minute peerTable := peerTableCache if peerTable == "" { + res, err := ps.ReloadPolicyFile(client) + if err != nil { + redirectWithError(w, r, "/config?", err) + return + } + + allowlistedPeers := res.GetAllowlistedPeers() + suspiciousPeers := res.GetSuspiciousPeerList() + + res2, err := ps.ListPeers(client) + if err != nil { + redirectWithError(w, r, "/config?", err) + return + } + peers = res2.GetPeers() + peerTable = convertPeersToHTMLTable(peers, allowlistedPeers, suspiciousPeers, swaps) peerTableCache = peerTable } @@ -298,6 +286,20 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { // use cache for page refreshes < 1 minute nonPeerTable = nonPeerTableCache if nonPeerTable == "" { + // make a list of peerswap peers + var psIds []string + + for _, peer := range peers { + psIds = append(psIds, peer.NodeId) + } + + // Get the remaining Lightning peers + res5, err := ln.ListPeers(cl, "", &psIds) + if err != nil { + redirectWithError(w, r, "/config?", err) + return + } + otherPeers := res5.GetPeers() nonPeerTable = convertOtherPeersToHTMLTable(otherPeers) nonPeerTableCache = nonPeerTable } From 6bfcc7049b0a53781fc7df815a9638e8d8c9e380 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 12:21:48 +0200 Subject: [PATCH 11/15] Forwards caching for cln --- CHANGELOG.md | 5 +- cmd/psweb/ln/cln.go | 132 +++++++++++++++------------- cmd/psweb/main.go | 4 +- cmd/psweb/templates/reusable.gohtml | 2 +- 4 files changed, 77 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd1910a..be0d324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ ## 1.4.1 -- Account to paid and received invoices -- Show circular rebalancing costs +- LND: Account for paid and received invoices in flows +- LND: Show LN costs of paid invoices (excluding peerswaps) +- Performance optimizations ## 1.4.0 diff --git a/cmd/psweb/ln/cln.go b/cmd/psweb/ln/cln.go index 46d9c81..aa6d0c0 100644 --- a/cmd/psweb/ln/cln.go +++ b/cmd/psweb/ln/cln.go @@ -25,6 +25,22 @@ const ( fileRPC = "lightning-rpc" ) +type Forwarding struct { + CreatedIndex uint64 `json:"created_index"` + InChannel string `json:"in_channel"` + OutChannel string `json:"out_channel"` + OutMsat uint64 `json:"out_msat"` + FeeMsat uint64 `json:"fee_msat"` + ResolvedTime float64 `json:"resolved_time"` +} + +var ( + // arrays mapped per channel + forwardsIn = make(map[uint64][]Forwarding) + forwardsOut = make(map[uint64][]Forwarding) + forwardsLastIndex uint64 +) + func GetClient() (*glightning.Lightning, func(), error) { lightning := glightning.NewLightning() err := lightning.StartUp(fileRPC, config.Config.RpcHost) @@ -60,7 +76,6 @@ func ConfirmedWalletBalance(client *glightning.Lightning) int64 { totalAmount += int64(amountMsat / 1000) } } - return totalAmount } @@ -188,6 +203,7 @@ type UnreserveInputsResponse struct { } func BumpPeginFee(newFeeRate uint64) (*SentResult, error) { + client, clean, err := GetClient() if err != nil { log.Println("GetClient:", err) @@ -356,19 +372,6 @@ func (r *ListForwardsRequest) Name() string { return "listforwards" } -type Forwarding struct { - CreatedIndex uint64 `json:"created_index"` - InChannel string `json:"in_channel"` - OutChannel string `json:"out_channel"` - OutMsat uint64 `json:"out_msat"` - FeeMsat uint64 `json:"fee_msat"` - ResolvedTime float64 `json:"resolved_time"` -} - -var forwards struct { - Forwards []Forwarding `json:"forwards"` -} - // cache routing history per channel from cln func CacheForwards() { // refresh history @@ -383,9 +386,9 @@ func CacheForwards() { } start := uint64(0) - if len(forwards.Forwards) > 0 { + if forwardsLastIndex > 0 { // continue from the last index + 1 - start = forwards.Forwards[len(forwards.Forwards)-1].CreatedIndex + 1 + start = forwardsLastIndex + 1 } // get incremental history @@ -395,8 +398,16 @@ func CacheForwards() { Start: start, }, &newForwards) - // append to all history - forwards.Forwards = append(forwards.Forwards, newForwards.Forwards...) + n := len(newForwards.Forwards) + if n > 0 { + forwardsLastIndex = newForwards.Forwards[n-1].CreatedIndex + for _, f := range newForwards.Forwards { + chIn := ConvertClnToLndChannelId(f.InChannel) + chOut := ConvertClnToLndChannelId(f.OutChannel) + forwardsIn[chIn] = append(forwardsIn[chIn], f) + forwardsOut[chOut] = append(forwardsOut[chOut], f) + } + } } // get routing statistics for a channel @@ -423,34 +434,30 @@ func GetForwardingStats(lndChannelId uint64) *ForwardingStats { timestamp30d := float64(now.AddDate(0, 0, -30).Unix()) timestamp6m := float64(now.AddDate(0, -6, 0).Unix()) - channelId := ConvertLndToClnChannelId(lndChannelId) - - for _, e := range forwards.Forwards { - if e.OutChannel == channelId { - if e.ResolvedTime > timestamp6m { - amountOut6m += e.OutMsat - feeMsat6m += e.FeeMsat - if e.ResolvedTime > timestamp30d { - amountOut30d += e.OutMsat - feeMsat30d += e.FeeMsat - if e.ResolvedTime > timestamp7d { - amountOut7d += e.OutMsat - feeMsat7d += e.FeeMsat - } + for _, e := range forwardsOut[lndChannelId] { + if e.ResolvedTime > timestamp6m { + amountOut6m += e.OutMsat + feeMsat6m += e.FeeMsat + if e.ResolvedTime > timestamp30d { + amountOut30d += e.OutMsat + feeMsat30d += e.FeeMsat + if e.ResolvedTime > timestamp7d { + amountOut7d += e.OutMsat + feeMsat7d += e.FeeMsat } } } - if e.InChannel == channelId { - if e.ResolvedTime > timestamp6m { - amountIn6m += e.OutMsat - assistedMsat6m += e.FeeMsat - if e.ResolvedTime > timestamp30d { - amountIn30d += e.OutMsat - assistedMsat30d += e.FeeMsat - if e.ResolvedTime > timestamp7d { - amountIn7d += e.OutMsat - assistedMsat7d += e.FeeMsat - } + } + for _, e := range forwardsIn[lndChannelId] { + if e.ResolvedTime > timestamp6m { + amountIn6m += e.OutMsat + assistedMsat6m += e.FeeMsat + if e.ResolvedTime > timestamp30d { + amountIn30d += e.OutMsat + assistedMsat30d += e.FeeMsat + if e.ResolvedTime > timestamp7d { + amountIn7d += e.OutMsat + assistedMsat7d += e.FeeMsat } } } @@ -493,36 +500,32 @@ func GetForwardingStats(lndChannelId uint64) *ForwardingStats { } // forwarding stats for a channel since timestamp -func GetForwardingStatsSinceTS(lndChannelId uint64, timeStamp uint64) *ShortForwardingStats { - +func GetChannelStats(lndChannelId uint64, timeStamp uint64) *ChannelStats { var ( - result ShortForwardingStats + result ChannelStats amountOut uint64 amountIn uint64 feeMsat uint64 assistedMsat uint64 ) - channelId := ConvertLndToClnChannelId(lndChannelId) timeStampF := float64(timeStamp) - for _, e := range forwards.Forwards { - if e.InChannel == channelId { - if e.ResolvedTime > timeStampF { - amountOut += e.OutMsat - feeMsat += e.FeeMsat - } + for _, e := range forwardsOut[lndChannelId] { + if e.ResolvedTime > timeStampF { + amountOut += e.OutMsat + feeMsat += e.FeeMsat } - if e.OutChannel == channelId { - if e.ResolvedTime > timeStampF { - amountIn += e.OutMsat - assistedMsat += e.FeeMsat - } + } + for _, e := range forwardsIn[lndChannelId] { + if e.ResolvedTime > timeStampF { + amountIn += e.OutMsat + assistedMsat += e.FeeMsat } } - result.AmountOut = amountOut / 1000 - result.AmountIn = amountIn / 1000 + result.RoutedOut = amountOut / 1000 + result.RoutedIn = amountIn / 1000 result.FeeSat = feeMsat / 1000 result.AssistedFeeSat = assistedMsat / 1000 @@ -562,7 +565,6 @@ func GetChannelInfo(client *glightning.Lightning, lndChannelId uint64, nodeId st break } } - return info } @@ -723,3 +725,11 @@ func GetMyAlias() string { } return myNodeAlias } + +func CachePayments() { + //not implemented +} + +func CacheInvoices() { + //not implemented +} diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index a17dabf..62dc35d 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -2004,7 +2004,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers ppmCost = totalCost * 1_000_000 / totalPayments } - peerTable += "" + formatWithThousandSeparators(totalFees) + " " + peerTable += "" + formatWithThousandSeparators(totalFees) + " " peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "" @@ -2180,7 +2180,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { if totalPayments > 0 { ppmCost = totalCost * 1_000_000 / totalPayments } - peerTable += "" + formatWithThousandSeparators(totalFees) + " " + peerTable += "" + formatWithThousandSeparators(totalFees) + " " peerTable += "" + formatWithThousandSeparators(totalCost) + "" peerTable += "" peerTable += "Invite " diff --git a/cmd/psweb/templates/reusable.gohtml b/cmd/psweb/templates/reusable.gohtml index c1fd3f4..4187ceb 100644 --- a/cmd/psweb/templates/reusable.gohtml +++ b/cmd/psweb/templates/reusable.gohtml @@ -21,7 +21,7 @@
-

{{.MempoolFeeRate}} sat/vB

+

{{.MempoolFeeRate}} sat/vB

From c2a83557d8ca3e1457e64f29315f705c4d3c40e2 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 13:07:47 +0200 Subject: [PATCH 12/15] Bump swap ts by 2 minutes --- cmd/psweb/ln/lnd.go | 13 ++++++++----- cmd/psweb/main.go | 23 ++++++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index ad6cdb7..214b614 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -801,9 +801,12 @@ func CacheInvoices() { // only append settled htlcs for _, invoice := range res.Invoices { if invoice.State == lnrpc.Invoice_SETTLED { - for _, htlc := range invoice.Htlcs { - if htlc.State == lnrpc.InvoiceHTLCState_SETTLED { - invoiceHtlcs[htlc.ChanId] = append(invoiceHtlcs[htlc.ChanId], htlc) + // exclude peerswap-related + if len(invoice.Memo) < 8 || invoice.Memo[:8] != "peerswap" { + for _, htlc := range invoice.Htlcs { + if htlc.State == lnrpc.InvoiceHTLCState_SETTLED { + invoiceHtlcs[htlc.ChanId] = append(invoiceHtlcs[htlc.ChanId], htlc) + } } } } @@ -952,12 +955,12 @@ func GetChannelStats(channelId uint64, timeStamp uint64) *ChannelStats { } } for _, e := range invoiceHtlcs[channelId] { - if uint64(e.ResolveTime) > timeStamp { + if uint64(e.AcceptTime) > timeStamp { invoicedMsat += e.AmtMsat } } for _, e := range paymentHtlcs[channelId] { - if uint64(e.ResolveTimeNs) > timestampNs { + if uint64(e.AttemptTimeNs) > timestampNs { paidOutMsat += e.Route.TotalAmtMsat costMsat += e.Route.TotalFeesMsat } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 62dc35d..d4de04e 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -193,20 +193,20 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { } defer cleanup() - res3, err := ps.ListSwaps(client) + res, err := ps.ListSwaps(client) if err != nil { redirectWithError(w, r, "/config?", err) return } - swaps := res3.GetSwaps() + swaps := res.GetSwaps() - res4, err := ps.LiquidGetBalance(client) + res2, err := ps.LiquidGetBalance(client) if err != nil { redirectWithError(w, r, "/config?", err) return } - satAmount := res4.GetSatAmount() + satAmount := res2.GetSatAmount() // Lightning RPC client cl, clean, er := ln.GetClient() @@ -289,6 +289,14 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { // make a list of peerswap peers var psIds []string + if len(peers) == 0 { + res, err := ps.ListPeers(client) + if err != nil { + redirectWithError(w, r, "/config?", err) + return + } + peers = res.GetPeers() + } for _, peer := range peers { psIds = append(psIds, peer.NodeId) } @@ -1851,11 +1859,8 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers for _, swap := range swaps { if simplifySwapState(swap.State) == "success" && swapTimestamps[swap.LndChanId] < swap.CreatedAt { - extraSeconds := int64(300) // 5 mins to ignore lighting payments related to swap - if swap.Asset == "btc" { - extraSeconds = 1200 // 20 mins for BTC swaps - } - swapTimestamps[swap.LndChanId] = swap.CreatedAt + extraSeconds + // bump by 2 minutes to exclude peerswap-related payments + swapTimestamps[swap.LndChanId] = swap.CreatedAt + 120 } } From b794051334adf1212b2c11d0207a241bfc4f9409 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 13:24:15 +0200 Subject: [PATCH 13/15] hide zero costs --- cmd/psweb/main.go | 20 +++++++++++++++----- cmd/psweb/templates/peer.gohtml | 4 ++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index d4de04e..2d1dfe7 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -1860,7 +1860,12 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers for _, swap := range swaps { if simplifySwapState(swap.State) == "success" && swapTimestamps[swap.LndChanId] < swap.CreatedAt { // bump by 2 minutes to exclude peerswap-related payments - swapTimestamps[swap.LndChanId] = swap.CreatedAt + 120 + bump := int64(120) + if swap.Asset == "btc" { + // bump by 20 minutes for BTC + bump = int64(1200) + } + swapTimestamps[swap.LndChanId] = swap.CreatedAt + bump } } @@ -2009,8 +2014,10 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers ppmCost = totalCost * 1_000_000 / totalPayments } - peerTable += "" + formatWithThousandSeparators(totalFees) + " " - peerTable += "" + formatWithThousandSeparators(totalCost) + "" + peerTable += "" + formatWithThousandSeparators(totalFees) + "" + if totalCost > 0 { + peerTable += " -" + formatWithThousandSeparators(totalCost) + "" + } peerTable += "		" if stringIsInSlice("lbtc", peer.SupportedAssets) { @@ -2182,11 +2189,14 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { if totalForwardsOut > 0 { ppmRevenue = totalFees * 1_000_000 / totalForwardsOut } + peerTable += "" + formatWithThousandSeparators(totalFees) + " " if totalPayments > 0 { ppmCost = totalCost * 1_000_000 / totalPayments } - peerTable += "" + formatWithThousandSeparators(totalFees) + " " - peerTable += "" + formatWithThousandSeparators(totalCost) + "" + if totalCost > 0 { + peerTable += " -" + formatWithThousandSeparators(totalCost) + "" + } + peerTable += "" peerTable += "Invite " peerTable += "
" diff --git a/cmd/psweb/templates/peer.gohtml b/cmd/psweb/templates/peer.gohtml index 6eb708e..ffa3b24 100644 --- a/cmd/psweb/templates/peer.gohtml +++ b/cmd/psweb/templates/peer.gohtml @@ -204,8 +204,8 @@ Sincerely, {{fmt .Peer.AsSender.SwapsOut}} {{if gt .Peer.PaidFee 0}} - - ({{.Peer.PaidFee}}) + + ({{.Peer.PaidFee}}) {{end}} From d8d12eb864fe8cd60acdd4e60663ef2cafe0537c Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 16:52:22 +0200 Subject: [PATCH 14/15] Add invoices and payments stats for CLN --- CHANGELOG.md | 5 +- cmd/psweb/ln/cln.go | 125 +++++++++++++++++++++++++++++++- cmd/psweb/main.go | 17 +++-- cmd/psweb/templates/peer.gohtml | 4 +- 4 files changed, 140 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be0d324..ac75aa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,8 @@ ## 1.4.1 -- LND: Account for paid and received invoices in flows -- LND: Show LN costs of paid invoices (excluding peerswaps) -- Performance optimizations +- Account for paid and received invoices in channel flow statistics +- Show LN costs of paid invoices (excluding peerswaps) ## 1.4.0 diff --git a/cmd/psweb/ln/cln.go b/cmd/psweb/ln/cln.go index aa6d0c0..0e78a68 100644 --- a/cmd/psweb/ln/cln.go +++ b/cmd/psweb/ln/cln.go @@ -499,7 +499,74 @@ func GetForwardingStats(lndChannelId uint64) *ForwardingStats { return &result } -// forwarding stats for a channel since timestamp +// Payment represents the structure of the payment data +type Payment struct { + Destination string `json:"destination"` + PaymentHash string `json:"payment_hash"` + Status string `json:"status"` + CreatedAt uint64 `json:"created_at"` + CompletedAt uint64 `json:"completed_at"` + Preimage string `json:"preimage"` + AmountMsat uint64 `json:"amount_msat"` + AmountSentMsat uint64 `json:"amount_sent_msat"` +} + +// HTLC represents the structure of a single HTLC entry +type HTLC struct { + ShortChannelID string `json:"short_channel_id"` + ID int `json:"id"` + Expiry int `json:"expiry"` + Direction string `json:"direction"` + AmountMsat uint64 `json:"amount_msat"` + PaymentHash string `json:"payment_hash"` + State string `json:"state"` +} + +type Invoice struct { + Label string `json:"label"` + Status string `json:"status"` + AmountReceivedMsat uint64 `json:"amount_received_msat,omitempty"` + PaidAt uint64 `json:"paid_at,omitempty"` + PaymentPreimage string `json:"payment_preimage,omitempty"` + CreatedIndex uint64 `json:"created_index"` +} + +type ListInvoicesRequest struct { + PaymentHash string `json:"payment_hash"` +} + +func (r ListInvoicesRequest) Name() string { + return "listinvoices" +} + +type ListInvoicesResponse struct { + Invoices []Invoice `json:"invoices"` +} +type ListPaysRequest struct { + PaymentHash string `json:"payment_hash"` +} + +func (r ListPaysRequest) Name() string { + return "listpays" +} + +type ListPaysResponse struct { + Payments []Payment `json:"pays"` +} + +type ListHtlcsRequest struct { + ChannelId string `json:"id,omitempty"` +} + +func (r ListHtlcsRequest) Name() string { + return "listhtlcs" +} + +type ListHtlcsResponse struct { + HTLCs []HTLC `json:"htlcs"` +} + +// flow stats for a channel since timestamp func GetChannelStats(lndChannelId uint64, timeStamp uint64) *ChannelStats { var ( result ChannelStats @@ -507,8 +574,61 @@ func GetChannelStats(lndChannelId uint64, timeStamp uint64) *ChannelStats { amountIn uint64 feeMsat uint64 assistedMsat uint64 + paidOutMsat uint64 + invoicedMsat uint64 + costMsat uint64 ) + channelId := ConvertLndToClnChannelId(lndChannelId) + + client, clean, err := GetClient() + if err != nil { + return &result + } + defer clean() + + var res ListHtlcsResponse + + err = client.Request(&ListHtlcsRequest{ + ChannelId: channelId, + }, &res) + if err != nil { + log.Println("ListHtlcsRequest:", err) + } + + for _, htlc := range res.HTLCs { + switch htlc.State { + case "SENT_REMOVE_ACK_REVOCATION": + // direction in, look for invoices + var inv ListInvoicesResponse + err := client.Request(&ListInvoicesRequest{ + PaymentHash: htlc.PaymentHash, + }, &inv) + if err == nil && + len(inv.Invoices) == 1 && + inv.Invoices[0].Status == "paid" && + inv.Invoices[0].PaidAt > timeStamp && + inv.Invoices[0].Label[:8] != "peerswap" { + invoicedMsat += htlc.AmountMsat + } + + case "RCVD_REMOVE_ACK_REVOCATION": + // direction out, look for payments + var pmt ListPaysResponse + err := client.Request(&ListPaysRequest{ + PaymentHash: htlc.PaymentHash, + }, &pmt) + if err == nil && + len(pmt.Payments) == 1 && + pmt.Payments[0].Status == "complete" && + pmt.Payments[0].CompletedAt > timeStamp { + paidOutMsat += htlc.AmountMsat + fee := pmt.Payments[0].AmountSentMsat - pmt.Payments[0].AmountMsat + costMsat += fee + } + } + } + timeStampF := float64(timeStamp) for _, e := range forwardsOut[lndChannelId] { @@ -528,6 +648,9 @@ func GetChannelStats(lndChannelId uint64, timeStamp uint64) *ChannelStats { result.RoutedIn = amountIn / 1000 result.FeeSat = feeMsat / 1000 result.AssistedFeeSat = assistedMsat / 1000 + result.InvoicedIn = invoicedMsat / 1000 + result.PaidOut = paidOutMsat / 1000 + result.PaidCost = costMsat / 1000 return &result } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 2d1dfe7..5907d3f 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -482,6 +482,7 @@ func peerHandler(w http.ResponseWriter, r *http.Request) { ChannelInfo []*ln.ChanneInfo PeerSwapPeer bool MyAlias string + FeePPM uint64 } feeRate := liquid.GetMempoolMinFee() @@ -489,6 +490,11 @@ func peerHandler(w http.ResponseWriter, r *http.Request) { feeRate = mempoolFeeRate } + lnFeePPM := uint64(0) + if peer.AsSender.SatsOut > 0 { + lnFeePPM = peer.PaidFee * 1_000_000 / peer.AsSender.SatsOut + } + data := Page{ ErrorMessage: errorMessage, PopUpMessage: "", @@ -510,6 +516,7 @@ func peerHandler(w http.ResponseWriter, r *http.Request) { ChannelInfo: channelInfo, PeerSwapPeer: psPeer, MyAlias: ln.GetMyAlias(), + FeePPM: lnFeePPM, } // executing template named "peer" @@ -1376,7 +1383,7 @@ func redirectWithError(w http.ResponseWriter, r *http.Request, redirectUrl strin t := fmt.Sprintln(err) // translate common errors into plain English switch { - case strings.HasPrefix(t, "rpc error: code = Unavailable desc = connection error"): + case strings.HasPrefix(t, "rpc error"): t = "Cannot connect to peerswapd. It either has not started listening yet or PeerSwap Host parameter is wrong. Check logs." case strings.HasPrefix(t, "Unable to dial socket"): t = "Cannot connect to lightningd. It either failed to start or has wrong configuration. Check logs." @@ -1862,8 +1869,8 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers // bump by 2 minutes to exclude peerswap-related payments bump := int64(120) if swap.Asset == "btc" { - // bump by 20 minutes for BTC - bump = int64(1200) + // bump by 60 minutes for BTC + bump = int64(3600) } swapTimestamps[swap.LndChanId] = swap.CreatedAt + bump } @@ -2016,7 +2023,7 @@ func convertPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer, allowlistedPeers peerTable += "" + formatWithThousandSeparators(totalFees) + "" if totalCost > 0 { - peerTable += " -" + formatWithThousandSeparators(totalCost) + "" + peerTable += " -" + formatWithThousandSeparators(totalCost) + "" } peerTable += "" @@ -2194,7 +2201,7 @@ func convertOtherPeersToHTMLTable(peers []*peerswaprpc.PeerSwapPeer) string { ppmCost = totalCost * 1_000_000 / totalPayments } if totalCost > 0 { - peerTable += " -" + formatWithThousandSeparators(totalCost) + "" + peerTable += " -" + formatWithThousandSeparators(totalCost) + "" } peerTable += "" diff --git a/cmd/psweb/templates/peer.gohtml b/cmd/psweb/templates/peer.gohtml index ffa3b24..3fd2ae5 100644 --- a/cmd/psweb/templates/peer.gohtml +++ b/cmd/psweb/templates/peer.gohtml @@ -204,8 +204,8 @@ Sincerely, {{fmt .Peer.AsSender.SwapsOut}} {{if gt .Peer.PaidFee 0}} - - ({{.Peer.PaidFee}}) + + ({{fmt .Peer.PaidFee}}) {{end}} From 7508e80c8d6c37105ae45e8ed454510254406c82 Mon Sep 17 00:00:00 2001 From: Impa10r Date: Sat, 18 May 2024 16:58:52 +0200 Subject: [PATCH 15/15] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac75aa4..71c7a21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 1.4.1 - Account for paid and received invoices in channel flow statistics -- Show LN costs of paid invoices (excluding peerswaps) +- Show Lightning costs of paid invoices (excluding peerswap's) ## 1.4.0