basic implementation of the last api version

the meat is in Noxer struct, in noxy.go.
executable server entry point is in cmd/noxy/main.go.

all request responses are cached using a rudimentary filesystem
based caching. a max storage quota is not implemented yet.
alex 2 years ago
parent 8dc6a62eb8
commit 23dd93f12a
Signed by: x1ddos
GPG Key ID: FDEFB4A63CBD8460

@ -123,6 +123,35 @@ now, fetch the preview image. note the change from `/meta` to `/data` endpoint:
[event raw data →](https://nostr.com/e/af30dac1d800acc25b87d0d6d0dd33bddf49e7f356556540a6c7722e3cb363fe)
## development
the binary's entry point is [cmd/noxy/main.go](cmd/noxy/main.go).
it imports packages in the root of the repo. the actual proxy is implemented
by `Noxer` in [noxy.go](noxy.go).
running the server locally:
mkdir cache
go run ./cmd/noxy -cachedir $PWD/cache
before sending a patch, make sure the code is passing tests:
go test -race
files are formatted:
go fmt ./...
and the go module file is updated:
go mod tidy
## release
a release binary is built using the following script.
it produces `noxy` executable in the root of the repo.
./tools/release.sh
before making a new release, you'll probably want to create a new git tag.
the tag is used as the noxy version, also printed when `-V` flag is specified.

@ -0,0 +1,217 @@
package noxy
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"io/ioutil"
"mime"
"os"
"path/filepath"
)
var ErrCacheMiss = errors.New("cache miss")
// DataStream is an io.Reader augmented with a mime type.
type DataStream struct {
ContentType string
r io.Reader
}
// MimeType reports the mime type as parsed from ds.ContentType, ignoring
// any errors resulting from parsing optional media parameters.
func (ds DataStream) MimeType() string {
mtype, _, err := mime.ParseMediaType(ds.ContentType)
if err != nil && errors.Is(err, mime.ErrInvalidMediaParameter) {
return ds.ContentType
}
return mtype
}
func (ds DataStream) Read(p []byte) (n int, err error) {
return ds.r.Read(p)
}
func (ds DataStream) Close() error {
if closer, ok := ds.r.(io.Closer); ok {
return closer.Close()
}
return nil
}
// Cacher is used by Noxer to store and use meta info in JSON format, and stream files.
type Cacher interface {
GetJSON(ctx context.Context, key CacheKey, dst any) error
PutJSON(ctx context.Context, key CacheKey, v any) error
GetStream(ctx context.Context, key CacheKey) (*DataStream, error)
PutStream(ctx context.Context, key CacheKey, mimeType string, r io.Reader) error
}
// CacheKeyType allows CacheKey to segregate data based on their logical types.
type CacheKeyType byte
const (
_ CacheKeyType = 1 << iota
CacheKeyEvent // nostr event links
CacheKeyURLPreview // OG link preview metadata
CacheKeyData // actual url data
)
// CacheKey is 33 bytes long, encoding its CacheKeyType at index 32.
type CacheKey []byte
// MakeCacheKey creates a new cache key based on sha256 of s and the logical data type.
func MakeCacheKey(s string, typ CacheKeyType) CacheKey {
h := sha256.Sum256([]byte(s))
return append(h[:], byte(typ))
}
// String returns a key representation without its CacheKeyType.
func (k CacheKey) String() string {
return hex.EncodeToString(k[:len(k)-1])
}
// Path result is suitable as a filesystem path for storing a cache entry.
func (k CacheKey) Path() string {
h := hex.EncodeToString(k[:len(k)-1])
return filepath.Join(k.Namespace(), h[0:4], h)
}
// Namespace is a string representation of the key's CacheKeyType.
func (k CacheKey) Namespace() string {
typ := byte(0)
if n := len(k); n > 0 {
typ = k[n-1]
}
switch CacheKeyType(typ) {
default:
return ""
case CacheKeyEvent:
return "event"
case CacheKeyURLPreview:
return "preview"
case CacheKeyData:
return "data"
}
}
// DirCache implements Cacher using regular filesystem operations.
// it places all data under subdirectories of the Root.
//
// TODO: cap at max storage size
// TODO: max size per key
type DirCache struct {
Root string
}
const dirCacheMetaSuffix = ".meta.json"
func (d DirCache) makeFilepath(key CacheKey, mkdir bool) string {
p := filepath.Join(d.Root, key.Path())
if mkdir {
os.MkdirAll(filepath.Dir(p), 0700)
}
return p
}
func (d DirCache) makeTemp(key CacheKey) string {
p := filepath.Join(d.Root, "tmp", key.Path())
os.MkdirAll(filepath.Dir(p), 0700)
return p
}
func (d DirCache) GetJSON(ctx context.Context, key CacheKey, dst any) error {
b, err := os.ReadFile(d.makeFilepath(key, false))
switch {
case err != nil && errors.Is(err, os.ErrNotExist):
return ErrCacheMiss
case err != nil:
return err
}
return json.Unmarshal(b, dst)
}
func (d DirCache) PutJSON(ctx context.Context, key CacheKey, v any) error {
b, err := json.Marshal(v)
if err != nil {
return err
}
return ioutil.WriteFile(d.makeFilepath(key, true), b, 0600)
}
func (d DirCache) GetStream(ctx context.Context, key CacheKey) (*DataStream, error) {
filepath := d.makeFilepath(key, false)
f, err := os.Open(filepath)
switch {
case err != nil && errors.Is(err, os.ErrNotExist):
return nil, ErrCacheMiss
case err != nil:
return nil, err
}
mb, err := os.ReadFile(filepath + dirCacheMetaSuffix)
if err != nil {
f.Close()
return nil, err
}
var ds DataStream
if err := json.Unmarshal(mb, &ds); err != nil {
f.Close()
return nil, err
}
ds.r = f
return &ds, nil
}
func (d DirCache) PutStream(ctx context.Context, key CacheKey, mimeType string, r io.Reader) error {
ds := DataStream{ContentType: mimeType}
mb, err := json.Marshal(ds)
if err != nil {
return err
}
tmpfile := d.makeTemp(key)
tmpmeta := tmpfile + dirCacheMetaSuffix
if err := ioutil.WriteFile(tmpmeta, mb, 0600); err != nil {
return err
}
f, err := os.OpenFile(tmpfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
if _, err := io.Copy(f, r); err != nil {
f.Close()
return err
}
if err := f.Close(); err != nil {
return err
}
filepath := d.makeFilepath(key, true)
if err := os.Rename(tmpfile, filepath); err != nil {
return err
}
return os.Rename(tmpmeta, filepath+dirCacheMetaSuffix)
}
// NullCache stores no data.
var NullCache nullCache
type nullCache struct{}
func (nc nullCache) GetJSON(context.Context, CacheKey, any) error {
return ErrCacheMiss
}
func (nc nullCache) PutJSON(context.Context, CacheKey, any) error {
return nil
}
func (nc nullCache) GetStream(context.Context, CacheKey) (*DataStream, error) {
return nil, ErrCacheMiss
}
func (nc nullCache) PutStream(ctx context.Context, k CacheKey, mtype string, r io.Reader) error {
_, err := io.Copy(io.Discard, r)
return err
}

@ -0,0 +1,206 @@
package main
import (
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"time"
"git.qcode.ch/nostr/noxy"
)
var noxyVersion = "dev" // overwritten by linker flags in release build
var (
listenAddr = flag.String("addr", "127.0.0.1:8889", "listen address")
cacheDir = flag.String("cachedir", "/tmp", "absolute cache dir")
maxFileSize = flag.Int64("maxfilesize", 1<<23, "refuse to handle files larger than this, in bytes")
idleRelayTimeout = flag.Duration("idlerelaytimeout", 10*time.Minute, "remove relay connections after idling this long")
showVersion = flag.Bool("V", false, "print version and exit")
// the -relay flag, populated by parseRelayFlag.
// set to defaultKnownRelays if empty.
knownRelays []string
defaultKnownRelays = []string{
"nostr.x1ddos.ch",
// from https://nostr.info/relays/
"expensive-relay.fiatjaf.com",
"nostr-pub.semisol.dev",
"nostr-pub.wellorder.net",
"nostr-relay-dev.wlvs.space",
"nostr-relay.wlvs.space",
"nostr-verified.wellorder.net",
"nostr.bitcoiner.social",
"nostr.delo.software",
"nostr.ono.re",
"nostr.onsats.org",
"nostr.openchain.fr",
"nostr.oxtr.dev",
"nostr.rdfriedl.com",
"nostr.semisol.dev",
"nostr.zaprite.io",
"relay.cynsar.foundation",
"relay.damus.io",
"relay.farscapian.com",
"relay.minds.com",
"relay.nostr.info",
"relay.oldcity-bitcoiners.info",
"relay.sovereign-stack.org",
}
)
func init() {
flag.Func("relay", "a comma separated nostr relays noxy is allowed to connect to", parseRelayFlag)
sort.Strings(defaultKnownRelays)
}
func parseRelayFlag(v string) error {
for _, s := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' }) {
s = strings.TrimSpace(s)
if s == "" {
continue
}
u, err := url.Parse(s)
if err != nil {
return fmt.Errorf("invalid relay URL %s: %w", s, err)
}
host := u.Hostname()
if host == "" {
return fmt.Errorf("invalid relay URL %s: no hostname", s)
}
knownRelays = append(knownRelays, host)
}
return nil
}
func usage() {
w := flag.CommandLine.Output()
fmt.Fprintf(w, "usage of %s:\n", os.Args[0])
flag.PrintDefaults()
fmt.Fprintln(w, "\nthe -relay flag may be specified multiple times.")
fmt.Fprintf(w, "its default value is the following list:\n\n")
fmt.Fprintln(w, strings.Join(defaultKnownRelays, "\n"))
}
// set up in main and used by handleXxx HTTP server handlers.
var noxer *noxy.Noxer
func main() {
flag.Usage = usage
flag.Parse()
if *showVersion {
fmt.Println(noxyVersion)
return
}
if len(knownRelays) == 0 {
knownRelays = defaultKnownRelays
}
sort.Strings(knownRelays)
if !filepath.IsAbs(*cacheDir) {
log.Fatal("cache dir must be absolute path")
}
noxer = &noxy.Noxer{
MaxFileSize: *maxFileSize,
IdleRelayTimeout: *idleRelayTimeout,
KnownRelays: knownRelays,
Cache: noxy.DirCache{Root: *cacheDir},
HTTPClient: &http.Client{Transport: &http.Transport{
MaxIdleConns: 100,
MaxConnsPerHost: 2,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}},
}
mux := http.NewServeMux()
mux.Handle("/", http.HandlerFunc(handleRoot))
mux.Handle("/meta", http.HandlerFunc(handleMeta))
mux.Handle("/data", http.HandlerFunc(handleData))
log.Printf("listening on %s", *listenAddr)
log.Printf("known relays: %s", strings.Join(knownRelays, ", "))
http.ListenAndServe(*listenAddr, logHandler(mux))
}
// handles requests to /
func handleRoot(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "this is noxy version %s\nhttps://git.qcode.ch/nostr/noxy\n", noxyVersion)
}
// handles requests to /meta
func handleMeta(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
eventID := r.FormValue("id")
relayURL := r.FormValue("relay")
linkURL := r.FormValue("url")
meta, err := noxer.FetchLinkMeta(r.Context(), eventID, relayURL, linkURL)
if err != nil {
writeError(w, err)
return
}
res := struct {
Type string `json:"type"`
Title string `json:"title"`
Descr string `json:"descr"`
Images []string `json:"images"`
}{
Type: meta.Type,
Title: meta.Title,
Descr: meta.Description,
Images: meta.ImageURLs,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(res)
}
// handles requests to /data
func handleData(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
eventID := r.FormValue("id")
relayURL := r.FormValue("relay")
linkURL := r.FormValue("url")
ds, err := noxer.StreamLinkData(r.Context(), eventID, relayURL, linkURL)
if err != nil {
writeError(w, err)
return
}
defer ds.Close()
w.Header().Set("Content-Type", ds.ContentType)
io.Copy(w, ds)
}
func writeError(w http.ResponseWriter, err error) {
log.Printf("ERROR: %v", err)
w.Header().Set("Content-Type", "text/plain")
switch {
default:
w.WriteHeader(http.StatusBadRequest)
case errors.Is(err, noxy.ErrNotFound):
w.WriteHeader(http.StatusNotFound)
}
fmt.Fprint(w, err.Error())
}
func logHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s", r.Method, r.RequestURI)
h.ServeHTTP(w, r)
})
}

@ -0,0 +1,21 @@
module git.qcode.ch/nostr/noxy
go 1.18
require (
github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a
github.com/nbd-wtf/go-nostr v0.8.1
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
mvdan.cc/xurls/v2 v2.4.0
)
require (
github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/valyala/fastjson v1.6.3 // indirect
golang.org/x/exp v0.0.0-20221106115401-f9659909a136 // indirect
)

@ -0,0 +1,38 @@
github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8 h1:Xa6tp8DPDhdV+k23uiTC/GrAYOe4IdyJVKtob4KW3GA=
github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8/go.mod h1:ihkm1viTbO/LOsgdGoFPBSvzqvx7ibvkMzYp3CgtHik=
github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a h1:etIrTD8BQqzColk9nKRusM9um5+1q0iOEJLqfBMIK64=
github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a/go.mod h1:emQhSYTXqB0xxjLITTw4EaWZ+8IIQYw+kx9GqNUKdLg=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/nbd-wtf/go-nostr v0.8.1 h1:DCbLiF1r3xHKBQA1Noz+97ra/B9AcftTh9w+syg3KzM=
github.com/nbd-wtf/go-nostr v0.8.1/go.mod h1:IIT/16QZ/nzi5cgQFU2WJrezYPNRi0iNgiitYMiu8UQ=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
golang.org/x/exp v0.0.0-20221106115401-f9659909a136 h1:Fq7F/w7MAa1KJ5bt2aJ62ihqp9HDcRuyILskkpIAurw=
golang.org/x/exp v0.0.0-20221106115401-f9659909a136/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc=
mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg=

53
io.go

@ -0,0 +1,53 @@
package noxy
import (
"bufio"
"errors"
"io"
)
var ErrSizeLimitExceeded = errors.New("size limit exceeded")
// HardLimitReader reads from r up to max bytes.
// it returns ErrSizeLimitExceeded if the number of read bytes is equal or exceeds max.
func HardLimitReader(r io.Reader, max int64) io.Reader {
return &hardLimitReader{r, max}
}
type hardLimitReader struct {
r io.Reader
n int64 // remaining bytes
}
func (l *hardLimitReader) Read(p []byte) (n int, err error) {
if l.n <= 0 {
return 0, ErrSizeLimitExceeded
}
if int64(len(p)) > l.n {
p = p[0:l.n]
}
n, err = l.r.Read(p)
l.n -= int64(n)
return
}
// SinkTeeReader stops writing to w at the first encountered write error
// but continues to propagate reads. the underlying writer is always buffered.
func SinkTeeReader(r io.Reader, w io.Writer) io.Reader {
return &sinkTeeReader{r: r, w: bufio.NewWriter(w)}
}
type sinkTeeReader struct {
r io.Reader
w *bufio.Writer
werr error
}
func (st *sinkTeeReader) Read(p []byte) (n int, err error) {
n, err = st.r.Read(p)
if n > 0 && st.werr == nil {
_, st.werr = st.w.Write(p[:n])
st.werr = st.w.Flush()
}
return
}

@ -0,0 +1,37 @@
package noxy
import (
"bytes"
"testing"
"testing/iotest"
)
// ErrorWriter always returns Err on Write calls.
type ErrorWriter struct {
Err error
}
func (ew ErrorWriter) Write([]byte) (int, error) {
return 0, ew.Err
}
func TestSinkTeeReader(t *testing.T) {
const text = "hello"
var w bytes.Buffer
tee := SinkTeeReader(bytes.NewBufferString(text), &w)
if err := iotest.TestReader(tee, []byte(text)); err != nil {
t.Errorf("tee reader: %v", err)
}
if v := string(w.Bytes()); v != text {
t.Errorf("b2 = %q; want %q", v, text)
}
}
func TestSinkTeeReaderErrWriter(t *testing.T) {
const text = "hello"
tee := SinkTeeReader(bytes.NewBufferString(text), ErrorWriter{iotest.ErrTimeout})
if err := iotest.TestReader(tee, []byte(text)); err != nil {
t.Errorf("tee reader: %v", err)
}
}

@ -0,0 +1,593 @@
package noxy
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
"sort"
"strings"
"sync"
"time"
"github.com/dyatlov/go-opengraph/opengraph"
nostr "github.com/nbd-wtf/go-nostr"
xurls "mvdan.cc/xurls/v2"
)
var (
ErrNotFound = errors.New("event or resource not found")
ErrUnsupportedEventKind = errors.New("unsupported event kind")
ErrUnsupportedMimeType = errors.New("unsupported link mime type")
ErrUnsupportedRelay = errors.New("unsupported relay")
)
// LinkMeta contains metadata info about a URL.
// it is typically assembled from OGP (https://ogp.me) by Noxer.FetchLinkMeta.
type LinkMeta struct {
Type string // og:type
Title string // og:title
Description string // og:description
ImageURLs []string // og:image:secure_url or og:image:url
}
// Noxer can proxy link preview info and data streams.
// See FetchLinkMeta and StreamLinkData for details.
//
// while the only required field is Cache, a zero value of KnownRelays
// makes Noxer refuse to proxy any URLs.
type Noxer struct {
// Cache is used to store both link preview meta info and
// data streamed to clients. it must be non-nil for Noxer to be usable.
Cache Cacher
// Noxer refuses to work with web pages and data streams larger than this value.
MaxFileSize int64 // defaults to 1Mb
// how long to keep an open connection to a relay without any activity.
// an activity is any cache-miss call to FetchLinkMeta or StreamLinkData.
// connections to relays are used to verify whether a link is part of
// an event contents. see aforementioned methods for more details.
IdleRelayTimeout time.Duration // defaults to 1min
// Noxer connects only to those relays hostnames of which are specified here.
// in other words, slice elements are only hostname parts of relay URLs.
// KnownRelays must be sorted in ascending order.
KnownRelays []string
// HTTPClient is used to make HTTP connections when fetching link preview
// info and data streaming. when nil, http.DefaultClient is used.
HTTPClient *http.Client
// clients keeps track of nostr relay connections to clean them up
// and remove idle after IdleRelayTimeout.
clientsMu sync.Mutex
clients map[string]*relayClient
cleanupTimer *time.Timer
// slurpers keep track of ongoing HTTP requests, both link preview
// meta info and data streams.
slurpersMu sync.Mutex
slurpers map[string]chan struct{}
}
// relayClient wraps nostr.Relay with an additional timestamp
// indicating last use of the relay to keep track of all active relay
// connections and remove idle.
//
// lastUsed is updated every time Noxer.fetchNostrEvent is called.
type relayClient struct {
relay *nostr.Relay
lastUsed time.Time
}
// FetchLinkMeta requests the web page at link URL, parses it as HTML and returns
// metadata found in the contents. It refuses to parse remote responses with
// content-type other than text/html.
//
// link URL must be found in content field of the nostr event posted to the
// specified relay. FetchLinkMeta connects to the nostr relay at relayURL
// and sends a filter'ed request with ids field set to eventID.
// the received event contents are "grepped" for the value of link as is.
//
// relayURL's hostname must be an element of x.KnownRelays.
// remote must respond with HTTP 200 OK to the link URL.
//
// successfully parsed link URLs are cached using Cacher.PutJSON. so, subsequent
// calls should not hit the remote server again unless x.Cache fails.
// concurrent requests are suspended until the context or first call is done.
func (x *Noxer) FetchLinkMeta(ctx context.Context, eventID, relayURL, link string) (*LinkMeta, error) {
if err := x.verifyEventLink(ctx, eventID, relayURL, link, verifyNoMeta); err != nil {
return nil, fmt.Errorf("verifyEventLink: %w", err)
}
return x.slurpLinkMeta(ctx, link)
}
func (x *Noxer) slurpLinkMeta(ctx context.Context, link string) (*LinkMeta, error) {
// use cache here instead of directly in FetchLinkMeta to avoid
// hitting remotes in x.verifyEventLink as much as possible.
cacheKey := MakeCacheKey(link, CacheKeyURLPreview)
var meta LinkMeta
cacheErr := x.Cache.GetJSON(ctx, cacheKey, &meta)
if cacheErr == nil {
return &meta, nil
}
log.Printf("cache.getjson %s(%s): %v", link, cacheKey, cacheErr)
ds, err := x.detachedSlurpData(ctx, link)
if err != nil {
return nil, fmt.Errorf("detachedSlurpData: %w", err)
}
defer ds.Close()
if mtype := ds.MimeType(); mtype != "text/html" {
return nil, fmt.Errorf("%w: received %q, want text/html", ErrUnsupportedMimeType, mtype)
}
res, err := parseLinkMeta(ds)
if err != nil {
return nil, fmt.Errorf("parseLinkMeta: %w", err)
}
if err := x.Cache.PutJSON(ctx, cacheKey, res); err != nil {
log.Printf("cache.putjson %s(%s): %v", link, cacheKey, err)
}
return res, nil
}
// StreamLinkData opens an HTTP connection to link and streams the response back.
// while doing so, it also caches the reponse bytes using Cache.PutStream. so,
// subsequent calls should not hit the remote link again unless x.Cache fails.
//
// link URL must be found in "content" field of the nostr event posted to the
// specified relay. StreamLinkData connects to the nostr relay at relayURL
// and sends a filter'ed request with ids field set to eventID.
// for event kinds 1 (text note) and 42 (channel message), the event contents
// are simply "grepped" for the value of link as is.
// for event kinds 0 (set metadata), 40 (create channel) and 41 (set channel
// metadata) the link is checked against "picture" field.
//
// additionally, link URL may be one of LinkMeta.ImageURLs as returned by
// x.FetchLinkMeta to a call with the same eventID.
//
// relayURL's hostname must be an element of x.KnownRelays.
// remote must respond with HTTP 200 OK to the link URL.
//
// callers must close DataStream.
// concurrent requests are suspended until the context or first call is done.
func (x *Noxer) StreamLinkData(ctx context.Context, eventID, relayURL, link string) (*DataStream, error) {
if err := x.verifyEventLink(ctx, eventID, relayURL, link, verifyExpandMeta); err != nil {
return nil, err
}
cacheKey := MakeCacheKey(link, CacheKeyData)
ds, err := x.Cache.GetStream(ctx, cacheKey)
if err != nil {
log.Printf("cache.getstream %s(%s): %v", link, cacheKey, err)
ds, err = x.detachedSlurpData(ctx, link)
}
return ds, err
}
// detachedSlurpData always finishes data streaming from remote url, event if
// the returned DataStream is closed prematurely, to cache the bytes for subsequent calls.
func (x *Noxer) detachedSlurpData(ctx context.Context, url string) (*DataStream, error) {
// check whether there's an ongoing stream. if so, wait and use cache or fail.
cacheKey := MakeCacheKey(url, CacheKeyData)
cacheKeyStr := cacheKey.Path()
x.slurpersMu.Lock()
slurpCh, found := x.slurpers[cacheKeyStr]
if found {
// a previous call is already streaming.
// wait 'till they're done, because the stream is non-seekable,
// then get it from cache or fail.
x.slurpersMu.Unlock()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-slurpCh:
return x.Cache.GetStream(ctx, cacheKey)
}
} else {
// wouldn't need this branch if close(slurpCh) was done after x.slurpersMu.Lock()
// in the goroutine below.
// but it's so easy to miss in future code changes that i don't want to risk it:
// not a big deal to check the cache one more time.
// reconsider if performance here becomes a concern.
ds, err := x.Cache.GetStream(ctx, cacheKey)
if err == nil {
x.slurpersMu.Unlock()
return ds, nil
}
}
// no other goroutine is streaming; do it now and make others wait on slurpCh.
slurpCh = x.makeSlurperChan(cacheKeyStr)
x.slurpersMu.Unlock()
// assuming 1min is enough to download a file.
// this may be too short for large values of x.MaxFileSize.
// TODO: compute ctx based on x.MaxFileSize?
ctx, cancelHTTP := context.WithTimeout(context.Background(), time.Minute)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
cancelHTTP()
return nil, err
}
resp, err := x.httpClient().Do(req)
if err != nil {
cancelHTTP()
return nil, err
}
if resp.StatusCode != http.StatusOK {
cancelHTTP()
if resp.StatusCode == http.StatusNotFound {
return nil, ErrNotFound
}
return nil, fmt.Errorf("bad HTTP response %s: %s", url, resp.Status)
}
ctype := resp.Header.Get("Content-Type")
if ctype == "" {
// TODO: sniff using mime magic bytes?
ctype = "application/octet-stream"
}
// rout is returned to the caller, wout is tee'ed from resp.Body.
// if the caller closes rout, tee'ing to wout also stops.
rout, wout := io.Pipe()
go func() {
defer func() {
resp.Body.Close()
wout.Close()
cancelHTTP()
close(slurpCh)
x.slurpersMu.Lock()
delete(x.slurpers, cacheKeyStr)
x.slurpersMu.Unlock()
}()
// the std io.TeeReader wouldn't work since it reports errors on reads
// from tee as soon as writes to wout fail which is the case if the caller
// closes rout.
tee := SinkTeeReader(HardLimitReader(resp.Body, x.maxFileSize()), wout)
if err := x.Cache.PutStream(ctx, cacheKey, ctype, tee); err != nil {
log.Printf("cache.putstream %s: %v", cacheKey, err)
// TODO: don't close; io.copy(wout, resp.body) here on cache failures?
}
}()
return &DataStream{ContentType: ctype, r: rout}, nil
}
// expandMeta arg values for verifyEventLink
const (
verifyExpandMeta = true
verifyNoMeta = false
)
// verifyEventLink checks whether link URL is in a nostr event's content,
// or one of OGP link preview URLs if expandMeta is true.
func (x *Noxer) verifyEventLink(ctx context.Context, eventID, relayURL, link string, expandMeta bool) error {
if !x.whitelistedRelay(relayURL) {
return ErrUnsupportedRelay
}
eventURLs, err := x.fetchEventURLs(ctx, eventID, relayURL)
if err != nil {
return err
}
log.Printf("fetched event URLs: %q", eventURLs)
for _, u := range eventURLs {
if u == link {
return nil
}
}
if !expandMeta {
return ErrNotFound
}
// link not found in the event text/json.
// check URLs in OGP metadata for each suitable link found in the event.
for _, urlStr := range eventURLs {
u, err := url.Parse(urlStr)
if err != nil {
continue // invalid url
}
if ext := path.Ext(u.Path); ext != "" {
if !strings.HasSuffix(ext, "html") && !strings.HasSuffix(ext, "htm") {
continue // assume not an html page
}
}
meta, err := x.slurpLinkMeta(ctx, urlStr)
if err != nil {
log.Printf("verifyEventLink slurpLinkMeta(%s): %v", u, err)
continue
}
for _, imgURL := range meta.ImageURLs {
if imgURL == link {
return nil
}
}
}
return ErrNotFound
}
// fetchEventURLs returns all URLs found in a nostr event.
// it assumes the relay URL is already checked to match x.KnownRelays.
func (x *Noxer) fetchEventURLs(ctx context.Context, eventID, relayURL string) ([]string, error) {
// check whether there's an ongoing fetch. if so, wait and use cache or fail.
cacheKey := MakeCacheKey(eventID, CacheKeyEvent)
cacheKeyStr := cacheKey.Path()
x.slurpersMu.Lock()
slurpCh, found := x.slurpers[cacheKeyStr]
if found {
// a previous call is already fetching.
// wait 'till they're done, then get it from cache or fail.
x.slurpersMu.Unlock()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-slurpCh:
var urls []string
err := x.Cache.GetJSON(ctx, cacheKey, &urls)
return urls, err
}
} else {
// same reasoning as in detachedSlurpData.
// wouldn't need this branch if close(slurpCh) was done after x.slurpersMu.Lock()
// in the goroutine below. but it's too easy to miss in future code changes.
// checking cache one more time here is most likely insignificant when compared to
// opening a websocket to a nostr relay.
var urls []string
if err := x.Cache.GetJSON(ctx, cacheKey, &urls); err == nil {
x.slurpersMu.Unlock()
return urls, nil
}
}
// no other goroutine is fetching; do it now and make others wait on slurpCh.
slurpCh = x.makeSlurperChan(cacheKeyStr)
x.slurpersMu.Unlock()
defer func() {
close(slurpCh)
x.slurpersMu.Lock()
delete(x.slurpers, cacheKeyStr)
x.slurpersMu.Unlock()
}()
event, err := x.fetchNostrEvent(ctx, eventID, relayURL)
if err != nil {
return nil, err
}
var eventURLs []string
switch event.Kind {
default:
return nil, ErrUnsupportedEventKind
case nostr.KindTextNote, nostr.KindChannelMessage:
eventURLs = extractAcceptableURLs(event.Content)
case nostr.KindSetMetadata, nostr.KindChannelCreation, nostr.KindChannelMetadata:
var p struct{ Picture string }
if err := json.Unmarshal([]byte(event.Content), &p); err != nil {
return nil, err
}
if validURL(p.Picture) {
eventURLs = append(eventURLs, p.Picture)
}
}
if err := x.Cache.PutJSON(ctx, cacheKey, eventURLs); err != nil {
log.Printf("cache.putjson %s: %v", cacheKey, err)
}
return eventURLs, nil
}
// assuming relay is whitelisted
func (x *Noxer) fetchNostrEvent(ctx context.Context, eventID, relayURL string) (*nostr.Event, error) {
relay, err := x.relayConn(ctx, relayURL)
if err != nil {
return nil, err
}
var (
event *nostr.Event
fetchErr error
)
// assuming 10sec is more than enough for a simple filter'ed sub with a single
// event ID.
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
done := make(chan struct{})
go func() {
defer close(done)
f := nostr.Filter{IDs: []string{eventID}, Limit: 1}
sub := relay.Subscribe(nostr.Filters{f})
defer sub.Unsub()
select {
case e := <-sub.Events:
// e.CheckSignature() is already done by the client
event = &e
case <-ctx.Done():
fetchErr = ctx.Err()
}
}()
select {
case <-done:
return event, fetchErr
case <-ctx.Done():
return nil, ctx.Err()
}
}
// connect to a nostr relay at relayURL or reuse an existing conn.
// it blocks all other callers.
func (x *Noxer) relayConn(ctx context.Context, relayURL string) (*nostr.Relay, error) {
// check existing conn and reuse if found
relayURL = nostr.NormalizeURL(relayURL)
x.clientsMu.Lock()
defer x.clientsMu.Unlock()
if cl, ok := x.clients[relayURL]; ok {
// "touch" the last used to let cleanup timer know we aren't idling
cl.lastUsed = time.Now()
return cl.relay, nil
}
// none found. make a new conn.
var (
relay *nostr.Relay
connErr error
)
// assuming 10sec is more than enough to connect to a websocket.
connCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
done := make(chan struct{})
go func() {
// TODO: send a patch upstream for a nostr.RelayConnectContext(ctx, url)
relay, connErr = nostr.RelayConnect(relayURL)
close(done)
}()
select {
case <-connCtx.Done():
// unfortunately, this leaves the above goroutine hanging, and will keep
// piling up for non-responsive relays.
// can be solved with a nostr.RelayConnectContext.
return nil, connCtx.Err()
case <-done:
if connErr != nil {
return nil, connErr
}
}
if x.clients == nil {
x.clients = make(map[string]*relayClient)
}
x.clients[relayURL] = &relayClient{
relay: relay,
lastUsed: time.Now(),
}
// a self-cleanup goroutine to delete ourselves if relay reports conn errors.
go func() {
err := <-relay.ConnectionError
log.Printf("%s: closing due to: %v", relayURL, err)
x.clientsMu.Lock()
defer x.clientsMu.Unlock()
relay.Close()
delete(x.clients, relayURL)
}()
if x.cleanupTimer == nil {
x.cleanupTimer = time.AfterFunc(x.idleRelayTimeout(), x.cleanupRelayConn)
}
return relay, nil
}
// close and delete nostr relay connections idling for more than x.idleRelayTimeout().
func (x *Noxer) cleanupRelayConn() {
x.clientsMu.Lock()
defer x.clientsMu.Unlock()
for url, cl := range x.clients {
if time.Since(cl.lastUsed) > x.idleRelayTimeout() {
log.Printf("closing idle conn to %s", url)
cl.relay.Close()
delete(x.clients, url)
}
}
if len(x.clients) > 0 {
x.cleanupTimer = time.AfterFunc(time.Minute, x.cleanupRelayConn)
} else {
x.cleanupTimer = nil
}
}
// assumes x.slurpersMu is handled by the caller.
func (x *Noxer) makeSlurperChan(k string) chan struct{} {
if x.slurpers == nil {
x.slurpers = make(map[string]chan struct{})
}
ch := make(chan struct{})
x.slurpers[k] = ch
return ch
}
func (x *Noxer) httpClient() *http.Client {
if x.HTTPClient == nil {
return http.DefaultClient
}
return x.HTTPClient
}
func (x *Noxer) idleRelayTimeout() time.Duration {
if x.IdleRelayTimeout == 0 {
return time.Minute
}
return x.IdleRelayTimeout
}
func (x *Noxer) maxFileSize() int64 {
if x.MaxFileSize == 0 {
return 1 << 20 // 1Mb
}
return x.MaxFileSize
}
// whitelistedRelay reports whether a nostr relay at urlStr is in x.KnownRelays.
// it expects x.KnownRelays to be sorted in lexical order.
//
// only hostname of urlStr is checked against x.KnownRelays.
func (x *Noxer) whitelistedRelay(urlStr string) bool {
u, err := url.Parse(urlStr)
if err != nil {
return false
}
host := u.Hostname()
i := sort.SearchStrings(x.KnownRelays, host)
return i < len(x.KnownRelays) && x.KnownRelays[i] == host
}
// TODO: use oEmbed if OGP fails?
func parseLinkMeta(r io.Reader) (*LinkMeta, error) {
og := opengraph.NewOpenGraph()
if err := og.ProcessHTML(r); err != nil {
return nil, err
}
if len(og.Images) == 0 {
return nil, ErrNotFound
}
meta := &LinkMeta{
Type: og.Type,
Title: og.Title,
Description: og.Description,
ImageURLs: make([]string, 0, len(og.Images)),
}
for _, img := range og.Images {
u := img.SecureURL
if u == "" {
u = img.URL
}
if u == "" {
continue
}
meta.ImageURLs = append(meta.ImageURLs, u)
}
return meta, nil
}
// TODO: patch to extract only host/ip; no emails and such
var urlRegexp = xurls.Relaxed()
func extractAcceptableURLs(text string) []string {
var urls []string
for _, a := range urlRegexp.FindAllString(text, -1) {
if validURL(a) {
urls = append(urls, a)
}
}
return urls
}
func validURL(urlStr string) bool {
if urlStr == "" {
return false
}
u, err := url.Parse(urlStr)
if err != nil {
return false
}
if u.Hostname() == "" {
return false
}
return u.Scheme == "" || u.Scheme == "http" || u.Scheme == "https"
}

@ -0,0 +1,414 @@
package noxy
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"net/http"
"net/http/httptest"
"net/url"
"os"
"sync"
"testing"
"testing/iotest"
"time"
nostr "github.com/nbd-wtf/go-nostr"
"golang.org/x/net/websocket"
)
func TestDetachedSlurpDataCacheMiss(t *testing.T) {
const contents = "text file"
const ctype = "text/plain;charset=utf-8"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", ctype)
w.Write([]byte(contents))
}))
defer ts.Close()
var testURL = ts.URL + "/"
cache := DirCache{Root: t.TempDir()}
noxer := Noxer{Cache: cache, MaxFileSize: 1024}
for i := 1; i <= 2; i++ {
t.Run(fmt.Sprintf("slurp %d", i), func(t *testing.T) {
bgCtx := context.Background()
canceledCtx, cancel := context.WithCancel(bgCtx)
cancel() // slurp must run on a separate context
ds, err := noxer.detachedSlurpData(canceledCtx, testURL)
if err != nil {
t.Fatalf("noxer.detachedSlurpData: %v", err)
}
checkDataStream(t, ds, ctype, []byte(contents))
checkCachedDataFile(t, cache, testURL, []byte(contents))
cacheKey := MakeCacheKey(testURL, CacheKeyData)
cachedDS, err := cache.GetStream(bgCtx, cacheKey)
if err != nil {
t.Fatalf("cache.GetStream: %v", err)
}
checkDataStream(t, cachedDS, ctype, []byte(contents))
noxer.slurpersMu.Lock()
defer noxer.slurpersMu.Unlock()
if len(noxer.slurpers) > 0 {
t.Error("x.slurpers is not empty")
}
})
}
}
func TestDetachedSlurpDataClosedReader(t *testing.T) {
const ctype = "text/plain;charset=utf-8"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", ctype)
w.Write([]byte("foo"))
time.Sleep(time.Second)
w.Write([]byte("bar"))
}))
defer ts.Close()
var testURL = ts.URL + "/"
cache := DirCache{Root: t.TempDir()}
noxer := Noxer{Cache: cache, MaxFileSize: 1024}
ctx := context.Background()
ds1, err := noxer.detachedSlurpData(ctx, testURL)
if err != nil {
t.Fatalf("noxer.detachedSlurpData 1: %v", err)
}
ds1.r.(io.Closer).Close()
cacheKey := MakeCacheKey(testURL, CacheKeyData)
noxer.slurpersMu.Lock()
ch := noxer.slurpers[cacheKey.Path()]
noxer.slurpersMu.Unlock()
select {
case <-time.After(3 * time.Second):
t.Fatal("slurp took too long")
case <-ch:
}
ds2, err := cache.GetStream(ctx, cacheKey)
if err != nil {
t.Fatalf("cache.GetStream: %v", err)
}
checkDataStream(t, ds2, ctype, []byte("foobar"))
}
func TestSlurpLinkMeta(t *testing.T) {
var count int
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if count > 0 {
w.WriteHeader(http.StatusNotFound)
return
}
count += 1
w.Header().Set("Content-Type", "text/html")
fmt.Fprintln(w, `<html><head>
<meta property="og:type" content="article" />
<meta property="og:title" content="test title" />
<meta property="og:description" content="test descr" />
<meta property="og:image" content="http://unused:0/image.png" />
</head></html>`)
}))
defer ts.Close()
var testURL = ts.URL + "/"
cache := DirCache{Root: t.TempDir()}
noxer := Noxer{Cache: cache, MaxFileSize: 1024}
meta1, err := noxer.slurpLinkMeta(context.Background(), testURL)
if err != nil {
t.Fatalf("slurpLinkMeta 1: %v", err)
}
wantMeta := &LinkMeta{
Type: "article",
Title: "test title",
Description: "test descr",
ImageURLs: []string{"http://unused:0/image.png"},
}
compareLinkMeta(t, meta1, wantMeta)
// expected to be cached by now
meta2, err := noxer.slurpLinkMeta(context.Background(), testURL)
if err != nil {
t.Fatalf("slurpLinkMeta 2: %v", err)
}
compareLinkMeta(t, meta2, wantMeta)
}
func TestSlurpLinkMetaHTTPErr(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer ts.Close()
var testURL = ts.URL + "/"
noxer := Noxer{Cache: NullCache, MaxFileSize: 1024}
_, err := noxer.slurpLinkMeta(context.Background(), testURL)
if !errors.Is(err, ErrNotFound) {
t.Errorf("slurpLinkMeta err=%v; want ErrNotFound", err)
}
}
func TestVerifyEventLinkNoMeta(t *testing.T) {
priv := genNostrKey()
event := &nostr.Event{
CreatedAt: time.Now(),
Kind: nostr.KindTextNote,
Content: "text; http://unused:0/foo and http://unused:0/bar",
PubKey: nostrPubKey(priv),
}
if err := event.Sign(priv); err != nil {
t.Fatal(err)
}
trelay := ServeSingleEvent(t, event)
defer trelay.Close()
t.Logf("fake relay URL: %s", trelay.URL)
noxer := Noxer{
Cache: DirCache{Root: t.TempDir()},
MaxFileSize: 1024,
KnownRelays: []string{"127.0.0.1"},
IdleRelayTimeout: time.Minute,
}
tt := []struct {
url string
wantOK bool
}{
{"http://unused:0/foo", true},
{"http://unused:0/bar", true},
{"http://unused:0/", false},
{"http://example.org", false},
}
for _, tc := range tt {
t.Run(tc.url, func(t *testing.T) {
ctx := context.Background()
err := noxer.verifyEventLink(ctx, event.ID, trelay.URL, tc.url, verifyNoMeta)
switch {
case tc.wantOK && err != nil:
t.Errorf("verifyEventLink: %v", err)
case !tc.wantOK && err == nil:
t.Error("verifyEventLink returned nil error")
}
})
}
if subs := trelay.OpenSubs(); len(subs) > 0 {
t.Errorf("trelay.OpenSubs is not empty: %q", subs)
}
}
func TestFetchMetaAndStreamData(t *testing.T) {
var website *httptest.Server
website = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
default:
w.WriteHeader(http.StatusBadRequest)
t.Errorf("%s %s", r.Method, r.URL)
case "/":
w.Header().Set("Content-Type", "text/html")
fmt.Fprintf(w, `<html><head>
<meta property="og:image" content="%s/image.png" />
</head></html>`, website.URL)
case "/image.png":
w.Header().Set("Content-Type", "image/png")
w.Write([]byte{1, 2, 3})
}
}))
defer website.Close()
websiteRootURL := website.URL + "/"
websiteImageURL := website.URL + "/image.png"
priv := genNostrKey()
event := &nostr.Event{
CreatedAt: time.Now(),
Kind: nostr.KindTextNote,
Content: fmt.Sprintf("link to an html page with image: %s", websiteRootURL),
PubKey: nostrPubKey(priv),
}
if err := event.Sign(priv); err != nil {
t.Fatal(err)
}
trelay := ServeSingleEvent(t, event)
defer trelay.Close()
cache := DirCache{Root: t.TempDir()}
noxer := Noxer{
Cache: cache,
MaxFileSize: 1024,
KnownRelays: []string{"127.0.0.1"},
IdleRelayTimeout: time.Minute,
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
meta, err := noxer.FetchLinkMeta(ctx, event.ID, trelay.URL, websiteRootURL)
if err != nil {
t.Fatalf("FetchLinkMeta(%s): %v", websiteRootURL, err)
}
var cachedMeta LinkMeta
if err := cache.GetJSON(ctx, MakeCacheKey(websiteRootURL, CacheKeyURLPreview), &cachedMeta); err != nil {
t.Fatalf("cache.getjson: %v", err)
}
compareLinkMeta(t, meta, &cachedMeta)
ds, err := noxer.StreamLinkData(ctx, event.ID, trelay.URL, websiteImageURL)
if err != nil {
t.Fatalf("StreamLinkData(%s): %v", websiteImageURL, err)
}
checkDataStream(t, ds, "image/png", []byte{1, 2, 3})
checkCachedDataFile(t, cache, websiteImageURL, []byte{1, 2, 3})
}
func checkDataStream(t *testing.T, ds *DataStream, ctype string, contents []byte) {
t.Helper()
if err := iotest.TestReader(ds, contents); err != nil {
t.Errorf("data stream reader: %v", err)
}
if ds.ContentType != ctype {
t.Errorf("ds.ContentType = %q; want %q", ds.ContentType, ctype)
}
}
func checkCachedDataFile(t *testing.T, cache DirCache, origURL string, contents []byte) {
t.Helper()
cacheKey := MakeCacheKey(origURL, CacheKeyData)
b, err := os.ReadFile(cache.makeFilepath(cacheKey, false))
if err != nil {
t.Errorf("cache file read: %v", err)
}
if !bytes.Equal(b, contents) {
t.Errorf("cached bytes = %q; want %q", b, contents)
}
}
func compareLinkMeta(t *testing.T, actual, expected *LinkMeta) {
t.Helper()
if actual.Type != expected.Type {
t.Errorf("actual.Type = %q; want %q", actual.Type, expected.Type)
}
if actual.Title != expected.Title {
t.Errorf("actual.Title = %q; want %q", actual.Title, expected.Title)
}
if actual.Description != expected.Description {
t.Errorf("actual.Description = %q; want %q", actual.Description, expected.Description)
}
if len(actual.ImageURLs) != 1 || actual.ImageURLs[0] != expected.ImageURLs[0] {
t.Errorf("actual.ImageURLs = %q; want %q", actual.ImageURLs, expected.ImageURLs)
}
}
func genNostrKey() string {
k := nostr.GeneratePrivateKey()
if k == "" {
panic("nostr.GeneratePrivateKey returned empty string")
}
return k
}
func nostrPubKey(priv string) string {
pub, err := nostr.GetPublicKey(priv)
if err != nil {
panic(err.Error())
}
return pub
}
type FakeNostrRelay struct {
Event *nostr.Event
URL string
HTTPServer *httptest.Server
Mu sync.Mutex
Subs map[string]bool // id => true if still active; false for unsub'ed IDs
}
func (nr *FakeNostrRelay) Close() {
nr.HTTPServer.Close()
}
func (nr *FakeNostrRelay) OpenSubs() []string {
nr.Mu.Lock()
defer nr.Mu.Unlock()
var a []string
for k, open := range nr.Subs {
if open {
a = append(a, k)
}
}
return a
}
func nostrHandler(t *testing.T, nr *FakeNostrRelay) func(*websocket.Conn) {
return func(conn *websocket.Conn) {
for {
var req [3]any
if err := websocket.JSON.Receive(conn, &req); err != nil {
conn.Close()
return
}
switch req[0].(string) {
default:
t.Errorf("ws handler req[0]=%q; want REQ or CLOSE", req[0])
conn.Close()
return
case "CLOSE":
nr.Mu.Lock()
defer nr.Mu.Unlock()
nr.Subs[req[1].(string)] = false
return
case "REQ":
subid := req[1].(string)
nr.Mu.Lock()
nr.Subs[subid] = true
nr.Mu.Unlock()
filters := req[2].(map[string]any)
t.Logf("ws handler sub=%q, filters=%s", subid, filters)
if ids := filters["ids"].([]any); len(ids) != 1 || ids[0].(string) != nr.Event.ID {
t.Errorf("ws handler REQ filter ids=%q; want [%q]", ids, []string{nr.Event.ID})
}
if limit := filters["limit"].(float64); math.Abs(limit-1) > 0.00001 {
t.Errorf("ws handler REQ limit=%f; want 1", limit)
}
b, err := json.Marshal(nr.Event)
if err != nil {
t.Errorf("json.Marshal: %v", err)
conn.Close()
return
}
resp := fmt.Sprintf(`["EVENT", %q, %s]`, subid, b)
t.Logf("ws handler resp: %s", resp)
if err := websocket.Message.Send(conn, resp); err != nil {
t.Errorf("ws handler REQ write: %v", err)
}
}
}
}
}
func ServeSingleEvent(t *testing.T, event *nostr.Event) *FakeNostrRelay {
relay := &FakeNostrRelay{
Event: event,
Subs: make(map[string]bool),
}
relay.HTTPServer = httptest.NewServer(&websocket.Server{
Handshake: func(conf *websocket.Config, r *http.Request) error {
t.Logf("new handshake from %s", r.RemoteAddr)
return nil
},
Handler: nostrHandler(t, relay),
})
tsurl, err := url.Parse(relay.HTTPServer.URL)
if err != nil {
panic(err)
}
relay.URL = fmt.Sprintf("ws://%s/", tsurl.Host)
return relay
}

@ -0,0 +1,6 @@
#!/bin/sh
set -e
VERSION=${VERSION:-$(git describe --tags)}
export CGO_ENABLED=0
exec go build -ldflags "-s -w -X main.noxyVersion=$VERSION" -buildmode=pie -trimpath ./cmd/noxy/
Loading…
Cancel
Save