basic implementation of the last api version

the meat is in Noxer struct, in noxy.go.
executable server entry point is in cmd/noxy/main.go.

all request responses are cached using a rudimentary filesystem
based caching. a max storage quota is not implemented yet.
pull/2/head
alex 1 year ago
parent 8dc6a62eb8
commit 813d0501bd
Signed by: x1ddos
GPG Key ID: FDEFB4A63CBD8460

@ -50,7 +50,7 @@ and `og:description`, respectively. future versions may also employ
[oEmbed](https://oembed.com/) and other metadata formats parsing.
the web page at `url` must be served with `text/html` content type. if the page
contains no or empty OGP metadata, noxy response with 200 OK and blank fields.
contains no or empty OGP metadata, noxy responds with 200 OK and blank fields.
otherwise, the response is a 4xx status code.
### /data
@ -123,6 +123,35 @@ now, fetch the preview image. note the change from `/meta` to `/data` endpoint:
[event raw data →](https://nostr.com/e/af30dac1d800acc25b87d0d6d0dd33bddf49e7f356556540a6c7722e3cb363fe)
## development
the binary's entry point is [cmd/noxy/main.go](cmd/noxy/main.go).
it imports packages in the root of the repo. the actual proxy is implemented
by `Noxer` in [noxy.go](noxy.go).
running the server locally:
mkdir cache
go run ./cmd/noxy -cachedir $PWD/cache
before sending a patch, make sure the code is passing tests:
go test -race
files are formatted:
go fmt ./...
and the go module file is updated:
go mod tidy
## release
a release binary is built using the following script.
it produces `noxy` executable in the root of the repo.
./tools/release.sh
before making a new release, you'll probably want to create a new git tag.
the tag is used as the noxy version, also printed when `-V` flag is specified.

@ -0,0 +1,217 @@
package noxy
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"io/ioutil"
"mime"
"os"
"path/filepath"
)
var ErrCacheMiss = errors.New("cache miss")
// DataStream is an io.Reader augmented with a mime type.
type DataStream struct {
ContentType string
r io.Reader
}
// MimeType reports the mime type as parsed from ds.ContentType, ignoring
// any errors resulting from parsing optional media parameters.
func (ds DataStream) MimeType() string {
mtype, _, err := mime.ParseMediaType(ds.ContentType)
if err != nil && errors.Is(err, mime.ErrInvalidMediaParameter) {
return ds.ContentType
}
return mtype
}
func (ds DataStream) Read(p []byte) (n int, err error) {
return ds.r.Read(p)
}
func (ds DataStream) Close() error {
if closer, ok := ds.r.(io.Closer); ok {
return closer.Close()
}
return nil
}
// Cacher is used by Noxer to store and use meta info in JSON format, and stream files.
type Cacher interface {
GetJSON(ctx context.Context, key CacheKey, dst any) error
PutJSON(ctx context.Context, key CacheKey, v any) error
GetStream(ctx context.Context, key CacheKey) (*DataStream, error)
PutStream(ctx context.Context, key CacheKey, mimeType string, r io.Reader) error
}
// CacheKeyType allows CacheKey to segregate data based on their logical types.
type CacheKeyType byte
const (
_ CacheKeyType = 1 << iota
CacheKeyEvent // nostr event links
CacheKeyURLPreview // OG link preview metadata
CacheKeyData // actual url data
)
// CacheKey is 33 bytes long, encoding its CacheKeyType at index 32.
type CacheKey []byte
// MakeCacheKey creates a new cache key based on sha256 of s and the logical data type.
func MakeCacheKey(s string, typ CacheKeyType) CacheKey {
h := sha256.Sum256([]byte(s))
return append(h[:], byte(typ))
}
// String returns a key representation without its CacheKeyType.
func (k CacheKey) String() string {
return hex.EncodeToString(k[:len(k)-1])
}
// Path result is suitable as a filesystem path for storing a cache entry.
func (k CacheKey) Path() string {
h := hex.EncodeToString(k[:len(k)-1])
return filepath.Join(k.Namespace(), h[0:4], h)
}
// Namespace is a string representation of the key's CacheKeyType.
func (k CacheKey) Namespace() string {
typ := byte(0)
if n := len(k); n > 0 {
typ = k[n-1]
}
switch CacheKeyType(typ) {
default:
return ""
case CacheKeyEvent:
return "event"
case CacheKeyURLPreview:
return "preview"
case CacheKeyData:
return "data"
}
}
// DirCache implements Cacher using regular filesystem operations.
// it places all data under subdirectories of the Root.
//
// TODO: cap at max storage size
// TODO: max size per key
type DirCache struct {
Root string
}
const dirCacheMetaSuffix = ".meta.json"
func (d DirCache) makeFilepath(key CacheKey, mkdir bool) string {
p := filepath.Join(d.Root, key.Path())
if mkdir {
os.MkdirAll(filepath.Dir(p), 0700)
}
return p
}
func (d DirCache) makeTemp(key CacheKey) string {
p := filepath.Join(d.Root, "tmp", key.Path())
os.MkdirAll(filepath.Dir(p), 0700)
return p
}
func (d DirCache) GetJSON(ctx context.Context, key CacheKey, dst any) error {
b, err := os.ReadFile(d.makeFilepath(key, false))
switch {
case err != nil && errors.Is(err, os.ErrNotExist):
return ErrCacheMiss
case err != nil:
return err
}
return json.Unmarshal(b, dst)
}
func (d DirCache) PutJSON(ctx context.Context, key CacheKey, v any) error {
b, err := json.Marshal(v)
if err != nil {
return err
}
return ioutil.WriteFile(d.makeFilepath(key, true), b, 0600)
}
func (d DirCache) GetStream(ctx context.Context, key CacheKey) (*DataStream, error) {
filepath := d.makeFilepath(key, false)
f, err := os.Open(filepath)
switch {
case err != nil && errors.Is(err, os.ErrNotExist):
return nil, ErrCacheMiss
case err != nil:
return nil, err
}
mb, err := os.ReadFile(filepath + dirCacheMetaSuffix)
if err != nil {
f.Close()
return nil, err
}
var ds DataStream
if err := json.Unmarshal(mb, &ds); err != nil {
f.Close()
return nil, err
}
ds.r = f
return &ds, nil
}
func (d DirCache) PutStream(ctx context.Context, key CacheKey, mimeType string, r io.Reader) error {
ds := DataStream{ContentType: mimeType}
mb, err := json.Marshal(ds)
if err != nil {
return err
}
tmpfile := d.makeTemp(key)
tmpmeta := tmpfile + dirCacheMetaSuffix
if err := ioutil.WriteFile(tmpmeta, mb, 0600); err != nil {
return err
}
f, err := os.OpenFile(tmpfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
if _, err := io.Copy(f, r); err != nil {
f.Close()
return err
}
if err := f.Close(); err != nil {
return err
}
filepath := d.makeFilepath(key, true)
if err := os.Rename(tmpfile, filepath); err != nil {
return err
}
return os.Rename(tmpmeta, filepath+dirCacheMetaSuffix)
}
// NullCache stores no data.
var NullCache nullCache
type nullCache struct{}
func (nc nullCache) GetJSON(context.Context, CacheKey, any) error {
return ErrCacheMiss
}
func (nc nullCache) PutJSON(context.Context, CacheKey, any) error {
return nil
}
func (nc nullCache) GetStream(context.Context, CacheKey) (*DataStream, error) {
return nil, ErrCacheMiss
}
func (nc nullCache) PutStream(ctx context.Context, k CacheKey, mtype string, r io.Reader) error {
_, err := io.Copy(io.Discard, r)
return err
}

@ -0,0 +1,206 @@
package main
import (
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"time"
"git.qcode.ch/nostr/noxy"
)
var noxyVersion = "dev" // overwritten by linker flags in release build
var (
listenAddr = flag.String("addr", "127.0.0.1:8889", "listen address")
cacheDir = flag.String("cachedir", "/tmp", "absolute cache dir")
maxFileSize = flag.Int64("maxfilesize", 1<<23, "refuse to handle files larger than this, in bytes")
idleRelayTimeout = flag.Duration("idlerelaytimeout", 10*time.Minute, "remove relay connections after idling this long")
showVersion = flag.Bool("V", false, "print version and exit")
// the -relay flag, populated by parseRelayFlag.
// set to defaultKnownRelays if empty.
knownRelays []string
defaultKnownRelays = []string{
"nostr.x1ddos.ch",
// from https://nostr.info/relays/
"expensive-relay.fiatjaf.com",
"nostr-pub.semisol.dev",
"nostr-pub.wellorder.net",
"nostr-relay-dev.wlvs.space",
"nostr-relay.wlvs.space",
"nostr-verified.wellorder.net",
"nostr.bitcoiner.social",
"nostr.delo.software",
"nostr.ono.re",
"nostr.onsats.org",
"nostr.openchain.fr",
"nostr.oxtr.dev",
"nostr.rdfriedl.com",
"nostr.semisol.dev",
"nostr.zaprite.io",
"relay.cynsar.foundation",
"relay.damus.io",
"relay.farscapian.com",
"relay.minds.com",
"relay.nostr.info",
"relay.oldcity-bitcoiners.info",
"relay.sovereign-stack.org",
}
)
func init() {
flag.Func("relay", "a comma separated nostr relays noxy is allowed to connect to", parseRelayFlag)
sort.Strings(defaultKnownRelays)
}
func parseRelayFlag(v string) error {
for _, s := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' }) {
s = strings.TrimSpace(s)
if s == "" {
continue
}
u, err := url.Parse(s)
if err != nil {
return fmt.Errorf("invalid relay URL %s: %w", s, err)
}
host := u.Hostname()
if host == "" {
return fmt.Errorf("invalid relay URL %s: no hostname", s)
}
knownRelays = append(knownRelays, host)
}
return nil
}
func usage() {
w := flag.CommandLine.Output()
fmt.Fprintf(w, "usage of %s:\n", os.Args[0])
flag.PrintDefaults()
fmt.Fprintln(w, "\nthe -relay flag may be specified multiple times.")
fmt.Fprintf(w, "its default value is the following list:\n\n")
fmt.Fprintln(w, strings.Join(defaultKnownRelays, "\n"))
}
// set up in main and used by handleXxx HTTP server handlers.
var noxer *noxy.Noxer
func main() {
flag.Usage = usage
flag.Parse()
if *showVersion {
fmt.Println(noxyVersion)
return
}
if len(knownRelays) == 0 {
knownRelays = defaultKnownRelays
}
sort.Strings(knownRelays)
if !filepath.IsAbs(*cacheDir) {
log.Fatal("cache dir must be absolute path")
}
noxer = &noxy.Noxer{
MaxFileSize: *maxFileSize,
IdleRelayTimeout: *idleRelayTimeout,
KnownRelays: knownRelays,
Cache: noxy.DirCache{Root: *cacheDir},
HTTPClient: &http.Client{Transport: &http.Transport{
MaxIdleConns: 100,
MaxConnsPerHost: 2,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}},
}
mux := http.NewServeMux()
mux.Handle("/", http.HandlerFunc(handleRoot))
mux.Handle("/meta", http.HandlerFunc(handleMeta))
mux.Handle("/data", http.HandlerFunc(handleData))
log.Printf("listening on %s", *listenAddr)
log.Printf("known relays: %s", strings.Join(knownRelays, ", "))
http.ListenAndServe(*listenAddr, logHandler(mux))
}
// handles requests to /
func handleRoot(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "this is noxy version %s\nhttps://git.qcode.ch/nostr/noxy\n", noxyVersion)
}
// handles requests to /meta
func handleMeta(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
eventID := r.FormValue("id")
relayURL := r.FormValue("relay")
linkURL := r.FormValue("url")
meta, err := noxer.FetchLinkMeta(r.Context(), eventID, relayURL, linkURL)
if err != nil {
writeError(w, err)
return
}
res := struct {
Type string `json:"type"`
Title string `json:"title"`
Descr string `json:"descr"`
Images []string `json:"images"`
}{
Type: meta.Type,
Title: meta.Title,
Descr: meta.Description,
Images: meta.ImageURLs,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(res)
}
// handles requests to /data
func handleData(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
eventID := r.FormValue("id")
relayURL := r.FormValue("relay")
linkURL := r.FormValue("url")
ds, err := noxer.StreamLinkData(r.Context(), eventID, relayURL, linkURL)
if err != nil {
writeError(w, err)
return
}
defer ds.Close()
w.Header().Set("Content-Type", ds.ContentType)
io.Copy(w, ds)
}
func writeError(w http.ResponseWriter, err error) {
log.Printf("ERROR: %v", err)
w.Header().Set("Content-Type", "text/plain")
switch {
default:
w.WriteHeader(http.StatusBadRequest)
case errors.Is(err, noxy.ErrNotFound):
w.WriteHeader(http.StatusNotFound)
}
fmt.Fprint(w, err.Error())
}
func logHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s", r.Method, r.RequestURI)
h.ServeHTTP(w, r)
})
}

@ -0,0 +1,21 @@
module git.qcode.ch/nostr/noxy
go 1.18
require (
github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a
github.com/nbd-wtf/go-nostr v0.8.1
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
mvdan.cc/xurls/v2 v2.4.0
)
require (
github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/valyala/fastjson v1.6.3 // indirect
golang.org/x/exp v0.0.0-20221106115401-f9659909a136 // indirect
)

@ -0,0 +1,38 @@
github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8 h1:Xa6tp8DPDhdV+k23uiTC/GrAYOe4IdyJVKtob4KW3GA=
github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8/go.mod h1:ihkm1viTbO/LOsgdGoFPBSvzqvx7ibvkMzYp3CgtHik=
github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a h1:etIrTD8BQqzColk9nKRusM9um5+1q0iOEJLqfBMIK64=
github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a/go.mod h1:emQhSYTXqB0xxjLITTw4EaWZ+8IIQYw+kx9GqNUKdLg=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/nbd-wtf/go-nostr v0.8.1 h1:DCbLiF1r3xHKBQA1Noz+97ra/B9AcftTh9w+syg3KzM=
github.com/nbd-wtf/go-nostr v0.8.1/go.mod h1:IIT/16QZ/nzi5cgQFU2WJrezYPNRi0iNgiitYMiu8UQ=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
golang.org/x/exp v0.0.0-20221106115401-f9659909a136 h1:Fq7F/w7MAa1KJ5bt2aJ62ihqp9HDcRuyILskkpIAurw=
golang.org/x/exp v0.0.0-20221106115401-f9659909a136/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc=
mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg=

53
io.go

@ -0,0 +1,53 @@
package noxy
import (
"bufio"
"errors"
"io"
)
var ErrSizeLimitExceeded = errors.New("size limit exceeded")
// HardLimitReader reads from r up to max bytes.
// it returns ErrSizeLimitExceeded if the number of read bytes is equal or exceeds max.
func HardLimitReader(r io.Reader, max int64) io.Reader {
return &hardLimitReader{r, max}
}
type hardLimitReader struct {
r io.Reader
n int64 // remaining bytes
}
func (l *hardLimitReader) Read(p []byte) (n int, err error) {
if l.n <= 0 {
return 0, ErrSizeLimitExceeded
}
if int64(len(p)) > l.n {
p = p[0:l.n]
}
n, err = l.r.Read(p)
l.n -= int64(n)
return
}
// SinkTeeReader stops writing to w at the first encountered write error
// but continues to propagate reads. the underlying writer is always buffered.
func SinkTeeReader(r io.Reader, w io.Writer) io.Reader {
return &sinkTeeReader{r: r, w: bufio.NewWriter(w)}
}
type sinkTeeReader struct {
r io.Reader
w *bufio.Writer
werr error
}
func (st *sinkTeeReader) Read(p []byte) (n int, err error) {
n, err = st.r.Read(p)
if n > 0 && st.werr == nil {
_, st.werr = st.w.Write(p[:n])
st.werr = st.w.Flush()
}
return
}

@ -0,0 +1,37 @@
package noxy
import (
"bytes"
"testing"
"testing/iotest"
)
// ErrorWriter always returns Err on Write calls.
type ErrorWriter struct {
Err error
}
func (ew ErrorWriter) Write([]byte) (int, error) {
return 0, ew.Err
}
func TestSinkTeeReader(t *testing.T) {
const text = "hello"
var w bytes.Buffer
tee := SinkTeeReader(bytes.NewBufferString(text), &w)
if err := iotest.TestReader(tee, []byte(text)); err != nil {
t.Errorf("tee reader: %v", err)
}
if v := string(w.Bytes()); v != text {
t.Errorf("b2 = %q; want %q", v, text)
}
}
func TestSinkTeeReaderErrWriter(t *testing.T) {
const text = "hello"
tee := SinkTeeReader(bytes.NewBufferString(text), ErrorWriter{iotest.ErrTimeout})
if err := iotest.TestReader(tee, []byte(text)); err != nil {
t.Errorf("tee reader: %v", err)
}
}

@ -0,0 +1,593 @@
package noxy
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
"sort"
"strings"
"sync"
"time"
"github.com/dyatlov/go-opengraph/opengraph"
nostr "github.com/nbd-wtf/go-nostr"
xurls "mvdan.cc/xurls/v2"
)
var (
ErrNotFound = errors.New("event or resource not found")
ErrUnsupportedEventKind = errors.New("unsupported event kind")
ErrUnsupportedMimeType = errors.New("unsupported link mime type")
ErrUnsupportedRelay = errors.New("unsupported relay")
)
// LinkMeta contains metadata info about a URL.
// it is typically assembled from OGP (https://ogp.me) by Noxer.FetchLinkMeta.
type LinkMeta struct {
Type string // og:type
Title string // og:title
Description string // og:description
ImageURLs []string // og:image:secure_url or og:image:url
}
// Noxer can proxy link preview info and data streams.
// See FetchLinkMeta and StreamLinkData for details.
//
// while the only required field is Cache, a zero value of KnownRelays
// makes Noxer refuse to proxy any URLs.
type Noxer struct {
// Cache is used to store both link preview meta info and
// data streamed to clients. it must be non-nil for Noxer to be usable.
Cache Cacher
// Noxer refuses to work with web pages and data streams larger than this value.
MaxFileSize int64 // defaults to 1Mb
// how long to keep an open connection to a relay without any activity.
// an activity is any cache-miss call to FetchLinkMeta or StreamLinkData.
// connections to relays are used to verify whether a link is part of
// an event contents. see aforementioned methods for more details.
IdleRelayTimeout time.Duration // defaults to 1min
// Noxer connects only to those relays hostnames of which are specified here.
// in other words, slice elements are only hostname parts of relay URLs.
// KnownRelays must be sorted in ascending order.
KnownRelays []string
// HTTPClient is used to make HTTP connections when fetching link preview
// info and data streaming. when nil, http.DefaultClient is used.
HTTPClient *http.Client
// clients keeps track of nostr relay connections to clean them up
// and remove idle after IdleRelayTimeout.
clientsMu sync.Mutex
clients map[string]*relayClient
cleanupTimer *time.Timer
// slurpers keep track of ongoing HTTP requests, both link preview
// meta info and data streams.
slurpersMu sync.Mutex
slurpers map[string]chan struct{}
}
// relayClient wraps nostr.Relay with an additional timestamp
// indicating last use of the relay to keep track of all active relay
// connections and remove idle.
//
// lastUsed is updated every time Noxer.fetchNostrEvent is called.
type relayClient struct {
relay *nostr.Relay
lastUsed time.Time
}
// FetchLinkMeta requests the web page at link URL, parses it as HTML and returns
// metadata found in the contents. It refuses to parse remote responses with
// content-type other than text/html.
//
// link URL must be found in content field of the nostr event posted to the
// specified relay. FetchLinkMeta connects to the nostr relay at relayURL
// and sends a filter'ed request with ids field set to eventID.
// the received event contents are "grepped" for the value of link as is.
//
// relayURL's hostname must be an element of x.KnownRelays.
// remote must respond with HTTP 200 OK to the link URL.
//
// successfully parsed link URLs are cached using Cacher.PutJSON. so, subsequent
// calls should not hit the remote server again unless x.Cache fails.
// concurrent requests are suspended until the context or first call is done.
func (x *Noxer) FetchLinkMeta(ctx context.Context, eventID, relayURL, link string) (*LinkMeta, error) {
if err := x.verifyEventLink(ctx, eventID, relayURL, link, verifyNoMeta); err != nil {
return nil, fmt.Errorf("verifyEventLink: %w", err)
}
return x.slurpLinkMeta(ctx, link)
}
func (x *Noxer) slurpLinkMeta(ctx context.Context, link string) (*LinkMeta, error) {
// use cache here instead of directly in FetchLinkMeta to avoid
// hitting remotes in x.verifyEventLink as much as possible.
cacheKey := MakeCacheKey(link, CacheKeyURLPreview)
var meta LinkMeta
cacheErr := x.Cache.GetJSON(ctx, cacheKey, &meta)
if cacheErr == nil {
return &meta, nil
}
log.Printf("cache.getjson %s(%s): %v", link, cacheKey, cacheErr)
ds, err := x.detachedSlurpData(ctx, link)
if err != nil {
return nil, fmt.Errorf("detachedSlurpData: %w", err)
}
defer ds.Close()
if mtype := ds.MimeType(); mtype != "text/html" {
return nil, fmt.Errorf("%w: received %q, want text/html", ErrUnsupportedMimeType, mtype)
}
res, err := parseLinkMeta(ds)
if err != nil {
return nil, fmt.Errorf("parseLinkMeta: %w", err)
}
if err := x.Cache.PutJSON(ctx, cacheKey, res); err != nil {
log.Printf("cache.putjson %s(%s): %v", link, cacheKey, err)
}
return res, nil
}
// StreamLinkData opens an HTTP connection to link and streams the response back.
// while doing so, it also caches the reponse bytes using Cache.PutStream. so,
// subsequent calls should not hit the remote link again unless x.Cache fails.
//
// link URL must be found in "content" field of the nostr event posted to the
// specified relay. StreamLinkData connects to the nostr relay at relayURL
// and sends a filter'ed request with ids field set to eventID.
// for event kinds 1 (text note) and 42 (channel message), the event contents
// are simply "grepped" for the value of link as is.
// for event kinds 0 (set metadata), 40 (create channel) and 41 (set channel
// metadata) the link is checked against "picture" field.
//
// additionally, link URL may be one of LinkMeta.ImageURLs as returned by
// x.FetchLinkMeta to a call with the same eventID.
//
// relayURL's hostname must be an element of x.KnownRelays.
// remote must respond with HTTP 200 OK to the link URL.
//
// callers must close DataStream.
// concurrent requests are suspended until the context or first call is done.
func (x *Noxer) StreamLinkData(ctx context.Context, eventID, relayURL, link string) (*DataStream, error) {
if err := x.verifyEventLink(ctx, eventID, relayURL, link, verifyExpandMeta); err != nil {
return nil, err
}
cacheKey := MakeCacheKey(link, CacheKeyData)
ds, err := x.Cache.GetStream(ctx, cacheKey)
if err != nil {
log.Printf("cache.getstream %s(%s): %v", link, cacheKey, err)
ds, err = x.detachedSlurpData(ctx, link)
}
return ds, err
}
// detachedSlurpData always finishes data streaming from remote url, event if
// the returned DataStream is closed prematurely, to cache the bytes for subsequent calls.
func (x *Noxer) detachedSlurpData(ctx context.Context, url string) (*DataStream, error) {
// check whether there's an ongoing stream. if so, wait and use cache or fail.
cacheKey := MakeCacheKey(url, CacheKeyData)
cacheKeyStr := cacheKey.Path()
x.slurpersMu.Lock()
slurpCh, found := x.slurpers[cacheKeyStr]
if found {
// a previous call is already streaming.
// wait 'till they're done, because the stream is non-seekable,
// then get it from cache or fail.
x.slurpersMu.Unlock()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-slurpCh:
return x.Cache.GetStream(ctx, cacheKey)
}
} else {
// wouldn't need this branch if close(slurpCh) was done after x.slurpersMu.Lock()
// in the goroutine below.
// but it's so easy to miss in future code changes that i don't want to risk it:
// not a big deal to check the cache one more time.
// reconsider if performance here becomes a concern.
ds, err := x.Cache.GetStream(ctx, cacheKey)
if err == nil {
x.slurpersMu.Unlock()
return ds, nil
}
}
// no other goroutine is streaming; do it now and make others wait on slurpCh.
slurpCh = x.makeSlurperChan(cacheKeyStr)
x.slurpersMu.Unlock()
// assuming 1min is enough to download a file.
// this may be too short for large values of x.MaxFileSize.
// TODO: compute ctx based on x.MaxFileSize?
ctx, cancelHTTP := context.WithTimeout(context.Background(), time.Minute)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
cancelHTTP()
return nil, err
}
resp, err := x.httpClient().Do(req)
if err != nil {
cancelHTTP()
return nil, err
}
if resp.StatusCode != http.StatusOK {
cancelHTTP()
if resp.StatusCode == http.StatusNotFound {
return nil, ErrNotFound
}
return nil, fmt.Errorf("bad HTTP response %s: %s", url, resp.Status)
}
ctype := resp.Header.Get("Content-Type")
if ctype == "" {
// TODO: sniff using mime magic bytes?
ctype = "application/octet-stream"
}
// rout is returned to the caller, wout is tee'ed from resp.Body.
// if the caller closes rout, tee'ing to wout also stops.
rout, wout := io.Pipe()
go func() {
defer func() {
resp.Body.Close()
wout.Close()
cancelHTTP()
close(slurpCh)
x.slurpersMu.Lock()
delete(x.slurpers, cacheKeyStr)
x.slurpersMu.Unlock()
}()
// the std io.TeeReader wouldn't work since it reports errors on reads
// from tee as soon as writes to wout fail which is the case if the caller
// closes rout.
tee := SinkTeeReader(HardLimitReader(resp.Body, x.maxFileSize()), wout)
if err := x.Cache.PutStream(ctx, cacheKey, ctype, tee); err != nil {
log.Printf("cache.putstream %s: %v", cacheKey, err)
// TODO: don't close; io.copy(wout, resp.body) here on cache failures?
}
}()
return &DataStream{ContentType: ctype, r: rout}, nil
}
// expandMeta arg values for verifyEventLink
const (
verifyExpandMeta = true
verifyNoMeta = false
)
// verifyEventLink checks whether link URL is in a nostr event's content,
// or one of OGP link preview URLs if expandMeta is true.
func (x *Noxer) verifyEventLink(ctx context.Context, eventID, relayURL, link string, expandMeta bool) error {
if !x.whitelistedRelay(relayURL) {
return ErrUnsupportedRelay
}
eventURLs, err := x.fetchEventURLs(ctx, eventID, relayURL)
if err != nil {
return err
}
log.Printf("fetched event URLs: %q", eventURLs)
for _, u := range eventURLs {
if u == link {
return nil
}
}
if !expandMeta {
return ErrNotFound
}
// link not found in the event text/json.
// check URLs in OGP metadata for each suitable link found in the event.
for _, urlStr := range eventURLs {
u, err := url.Parse(urlStr)
if err != nil {
continue // invalid url
}
if ext := path.Ext(u.Path); ext != "" {
if !strings.HasSuffix(ext, "html") && !strings.HasSuffix(ext, "htm") {
continue // assume not an html page
}
}
meta, err := x.slurpLinkMeta(ctx, urlStr)
if err != nil {
log.Printf("verifyEventLink slurpLinkMeta(%s): %v", u, err)
continue
}
for _, imgURL := range meta.ImageURLs {
if imgURL == link {
return nil
}
}
}
return ErrNotFound
}
// fetchEventURLs returns all URLs found in a nostr event.
// it assumes the relay URL is already checked to match x.KnownRelays.
func (x *Noxer) fetchEventURLs(ctx context.Context, eventID, relayURL string) ([]string, error) {
// check whether there's an ongoing fetch. if so, wait and use cache or fail.
cacheKey := MakeCacheKey(eventID, CacheKeyEvent)
cacheKeyStr := cacheKey.Path()
x.slurpersMu.Lock()
slurpCh, found := x.slurpers[cacheKeyStr]
if found {
// a previous call is already fetching.
// wait 'till they're done, then get it from cache or fail.
x.slurpersMu.Unlock()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-slurpCh:
var urls []string
err := x.Cache.GetJSON(ctx, cacheKey, &urls)
return urls, err
}
} else {
// same reasoning as in detachedSlurpData.
// wouldn't need this branch if close(slurpCh) was done after x.slurpersMu.Lock()
// in the goroutine below. but it's too easy to miss in future code changes.
// checking cache one more time here is most likely insignificant when compared to
// opening a websocket to a nostr relay.
var urls []string
if err := x.Cache.GetJSON(ctx, cacheKey, &urls); err == nil {
x.slurpersMu.Unlock()
return urls, nil
}
}
// no other goroutine is fetching; do it now and make others wait on slurpCh.
slurpCh = x.makeSlurperChan(cacheKeyStr)
x.slurpersMu.Unlock()
defer func() {
close(slurpCh)
x.slurpersMu.Lock()
delete(x.slurpers, cacheKeyStr)
x.slurpersMu.Unlock()
}()
event, err := x.fetchNostrEvent(ctx, eventID, relayURL)
if err != nil {
return nil, err
}
var eventURLs []string
switch event.Kind {
default:
return nil, ErrUnsupportedEventKind
case nostr.KindTextNote, nostr.KindChannelMessage:
eventURLs = extractAcceptableURLs(event.Content)
case nostr.KindSetMetadata, nostr.KindChannelCreation, nostr.KindChannelMetadata:
var p struct{ Picture string }
if err := json.Unmarshal([]byte(event.Content), &p); err != nil {
return nil, err
}
if validURL(p.Picture) {
eventURLs = append(eventURLs, p.Picture)
}
}
if err := x.Cache.PutJSON(ctx, cacheKey, eventURLs); err != nil {
log.Printf("cache.putjson %s: %v", cacheKey, err)
}
return eventURLs, nil
}
// assuming relay is whitelisted
func (x *Noxer) fetchNostrEvent(ctx context.Context, eventID, relayURL string) (*nostr.Event, error) {
relay, err := x.relayConn(ctx, relayURL)
if err != nil {
return nil, err
}
var (
event *nostr.Event
fetchErr error
)
// assuming 10sec is more than enough for a simple filter'ed sub with a single
// event ID.
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
done := make(chan struct{})
go func() {
defer close(done)
f := nostr.Filter{IDs: []string{eventID}, Limit: 1}
sub := relay.Subscribe(nostr.Filters{f})
defer sub.Unsub()
select {
case e := <-sub.Events:
// e.CheckSignature() is already done by the client
event = &e
case <-ctx.Done():
fetchErr = ctx.Err()
}
}()
select {
case <-done:
return event, fetchErr
case <-ctx.Done():
return nil, ctx.Err()
}
}
// connect to a nostr relay at relayURL or reuse an existing conn.
// it blocks all other callers.
func (x *Noxer) relayConn(ctx context.Context, relayURL string) (*nostr.Relay, error) {
// check existing conn and reuse if found
relayURL = nostr.NormalizeURL(relayURL)
x.clientsMu.Lock()
defer x.clientsMu.Unlock()
if cl, ok := x.clients[relayURL]; ok {
// "touch" the last used to let cleanup timer know we aren't idling
cl.lastUsed = time.Now()
return cl.relay, nil
}
// none found. make a new conn.
var (
relay *nostr.Relay
connErr error
)
// assuming 10sec is more than enough to connect to a websocket.
connCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
done := make(chan struct{})
go func() {
// TODO: send a patch upstream for a nostr.RelayConnectContext(ctx, url)
relay, connErr = nostr.RelayConnect(relayURL)
close(done)
}()
select {
case <-connCtx.Done():
// unfortunately, this leaves the above goroutine hanging, and will keep
// piling up for non-responsive relays.
// can be solved with a nostr.RelayConnectContext.
return nil, connCtx.Err()
case <-done:
if connErr != nil {
return nil, connErr
}
}
if x.clients == nil {
x.clients = make(map[string]*relayClient)
}
x.clients[relayURL] = &relayClient{
relay: relay,
lastUsed: time.Now(),
}
// a self-cleanup goroutine to delete ourselves if relay reports conn errors.
go func() {
err := <-relay.ConnectionError
log.Printf("%s: closing due to: %v", relayURL, err)
x.clientsMu.Lock()
defer x.clientsMu.Unlock()
relay.Close()
delete(x.clients, relayURL)
}()
if x.cleanupTimer == nil {
x.cleanupTimer = time.AfterFunc(x.idleRelayTimeout(), x.cleanupRelayConn)
}
return relay, nil
}
// close and delete nostr relay connections idling for more than x.idleRelayTimeout().
func (x *Noxer) cleanupRelayConn() {
x.clientsMu.Lock()
defer x.clientsMu.Unlock()
for url, cl := range x.clients {
if time.Since(cl.lastUsed) > x.idleRelayTimeout() {
log.Printf("closing idle conn to %s", url)
cl.relay.Close()
delete(x.clients, url)
}
}
if len(x.clients) > 0 {
x.cleanupTimer = time.AfterFunc(time.Minute, x.cleanupRelayConn)
} else {
x.cleanupTimer = nil
}
}
// assumes x.slurpersMu is handled by the caller.
func (x *Noxer) makeSlurperChan(k string) chan struct{} {
if x.slurpers == nil {
x.slurpers = make(map[string]chan struct{})
}
ch := make(chan struct{})
x.slurpers[k] = ch
return ch
}
func (x *Noxer) httpClient() *http.Client {
if x.HTTPClient == nil {
return http.DefaultClient
}
return x.HTTPClient
}
func (x *Noxer) idleRelayTimeout() time.Duration {
if x.IdleRelayTimeout == 0 {
return time.Minute
}
return x.IdleRelayTimeout
}
func (x *Noxer) maxFileSize() int64 {
if x.MaxFileSize == 0 {
return 1 << 20 // 1Mb
}
return x.MaxFileSize
}
// whitelistedRelay reports whether a nostr relay at urlStr is in x.KnownRelays.
// it expects x.KnownRelays to be sorted in lexical order.
//
// only hostname of urlStr is checked against x.KnownRelays.
func (x *Noxer) whitelistedRelay(urlStr string) bool {
u, err := url.Parse(urlStr)
if err != nil {
return false
}
host := u.Hostname()
i := sort.SearchStrings(x.KnownRelays, host)
return i < len(x.KnownRelays) && x.KnownRelays[i] == host
}
// TODO: use oEmbed if OGP fails?
func parseLinkMeta(r io.Reader) (*LinkMeta, error) {
og := opengraph.NewOpenGraph()
if err := og.ProcessHTML(r); err != nil {
return nil, err
}
if len(og.Images) == 0 {
return nil, ErrNotFound
}
meta := &LinkMeta{
Type: og.Type,
Title: og.Title,
Description: og.Description,
ImageURLs: make([]string, 0, len(og.Images)),
}
for _, img := range og.Images {
u := img.SecureURL
if u == "" {
u = img.URL
}
if u == "" {
continue
}
meta.ImageURLs = append(meta.ImageURLs, u)
}
return meta, nil
}
// TODO: patch to extract only host/ip; no emails and such
var urlRegexp = xurls.Relaxed()
func extractAcceptableURLs(text string) []string {
var urls []string
for _, a := range urlRegexp.FindAllString(text, -1) {
if validURL(a) {
urls = append(urls, a)
}
}
return urls
}
func validURL(urlStr string) bool {
if urlStr == "" {
return false
}
u, err := url.Parse(urlStr)
if err != nil {
return false
}
if u.Hostname() == "" {
return false
}
return u.Scheme == "" || u.Scheme == "http" || u.Scheme == "https"
}

@ -0,0 +1,414 @@
package noxy
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"net/http"
"net/http/httptest"
"net/url"
"os"
"sync"
"testing"
"testing/iotest"
"time"
nostr "github.com/nbd-wtf/go-nostr"
"golang.org/x/net/websocket"
)
func TestDetachedSlurpDataCacheMiss(t *testing.T) {
const contents = "text file"
const ctype = "text/plain;charset=utf-8"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", ctype)
w.Write([]byte(contents))
}))
defer ts.Close()
var testURL = ts.URL + "/"
cache := DirCache{Root: t.TempDir()}
noxer := Noxer{Cache: cache, MaxFileSize: 1024}
for i := 1; i <= 2; i++ {
t.Run(fmt.Sprintf("slurp %d", i), func(t *testing.T) {
bgCtx := context.Background()
canceledCtx, cancel := context.WithCancel(bgCtx)
cancel() // slurp must run on a separate context
ds, err := noxer.detachedSlurpData(canceledCtx, testURL)
if err != nil {
t.Fatalf("noxer.detachedSlurpData: %v", err)
}
checkDataStream(t, ds, ctype, []byte(contents))
checkCachedDataFile(t, cache, testURL, []byte(contents))
cacheKey := MakeCacheKey(testURL, CacheKeyData)
cachedDS, err := cache.GetStream(bgCtx, cacheKey)
if err != nil {
t.Fatalf("cache.GetStream: %v", err)
}
checkDataStream(t, cachedDS, ctype, []byte(contents))
noxer.slurpersMu.Lock()
defer noxer.slurpersMu.Unlock()
if len(noxer.slurpers) > 0 {
t.Error("x.slurpers is not empty")
}
})
}
}
func TestDetachedSlurpDataClosedReader(t *testing.T) {
const ctype = "text/plain;charset=utf-8"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", ctype)
w.Write([]byte("foo"))
time.Sleep(time.Second)
w.Write([]byte("bar"))
}))
defer ts.Close()
var testURL = ts.URL + "/"
cache := DirCache{Root: t.TempDir()}
noxer := Noxer{Cache: cache, MaxFileSize: 1024}
ctx := context.Background()
ds1, err := noxer.detachedSlurpData(ctx, testURL)
if err != nil {
t.Fatalf("noxer.detachedSlurpData 1: %v", err)
}
ds1.r.(io.Closer).Close()
cacheKey := MakeCacheKey(testURL, CacheKeyData)
noxer.slurpersMu.Lock()
ch := noxer.slurpers[cacheKey.Path()]
noxer.slurpersMu.Unlock()
select {
case <-time.After(3 * time.Second):
t.Fatal("slurp took too long")
case <-ch:
}
ds2, err := cache.GetStream(ctx, cacheKey)
if err != nil {
t.Fatalf("cache.GetStream: %v", err)
}
checkDataStream(t, ds2, ctype, []byte("foobar"))
}
func TestSlurpLinkMeta(t *testing.T) {
var count int
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if count > 0 {
w.WriteHeader(http.StatusNotFound)
return
}
count += 1
w.Header().Set("Content-Type", "text/html")
fmt.Fprintln(w, `<html><head>
<meta property="og:type" content="article" />
<meta property="og:title" content="test title" />
<meta property="og:description" content="test descr" />
<meta property="og:image" content="http://unused:0/image.png" />
</head></html>`)
}))
defer ts.Close()
var testURL = ts.URL + "/"
cache := DirCache{Root: t.TempDir()}
noxer := Noxer{Cache: cache, MaxFileSize: 1024}
meta1, err := noxer.slurpLinkMeta(context.Background(), testURL)
if err != nil {
t.Fatalf("slurpLinkMeta 1: %v", err)
}
wantMeta := &LinkMeta{
Type: "article",
Title: "test title",
Description: "test descr",
ImageURLs: []string{"http://unused:0/image.png"},
}
compareLinkMeta(t, meta1, wantMeta)
// expected to be cached by now
meta2, err := noxer.slurpLinkMeta(context.Background(), testURL)
if err != nil {
t.Fatalf("slurpLinkMeta 2: %v", err)
}
compareLinkMeta(t, meta2, wantMeta)
}
func TestSlurpLinkMetaHTTPErr(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer ts.Close()
var testURL = ts.URL + "/"
noxer := Noxer{Cache: NullCache, MaxFileSize: 1024}
_, err := noxer.slurpLinkMeta(context.Background(), testURL)
if !errors.Is(err, ErrNotFound) {
t.Errorf("slurpLinkMeta err=%v; want ErrNotFound", err)
}
}
func TestVerifyEventLinkNoMeta(t *testing.T) {
priv := genNostrKey()
event := &nostr.Event{
CreatedAt: time.Now(),
Kind: nostr.KindTextNote,
Content: "text; http://unused:0/foo and http://unused:0/bar",
PubKey: nostrPubKey(priv),
}
if err := event.Sign(priv); err != nil {
t.Fatal(err)
}
trelay := ServeSingleEvent(t, event)
defer trelay.Close()
t.Logf("fake relay URL: %s", trelay.URL)
noxer := Noxer{
Cache: DirCache{Root: t.TempDir()},
MaxFileSize: 1024,
KnownRelays: []string{"127.0.0.1"},
IdleRelayTimeout: time.Minute,
}
tt := []struct {
url string
wantOK bool
}{
{"http://unused:0/foo", true},
{"http://unused:0/bar", true},
{"http://unused:0/", false},
{"http://example.org", false},
}
for _, tc := range tt {
t.Run(tc.url, func(t *testing.T) {
ctx := context.Background()
err := noxer.verifyEventLink(ctx, event.ID, trelay.URL, tc.url, verifyNoMeta)
switch {
case tc.wantOK && err != nil:
t.Errorf("verifyEventLink: %v", err)
case !tc.wantOK && err == nil:
t.Error("verifyEventLink returned nil error")
}
})
}
if subs := trelay.OpenSubs(); len(subs) > 0 {
t.Errorf("trelay.OpenSubs is not empty: %q", subs)
}
}
func TestFetchMetaAndStreamData(t *testing.T) {
var website *httptest.Server
website = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
default:
w.WriteHeader(http.StatusBadRequest)
t.Errorf("%s %s", r.Method, r.URL)
case "/":
w.Header().Set("Content-Type", "text/html")
fmt.Fprintf(w, `<html><head>
<meta property="og:image" content="%s/image.png" />
</head></html>`, website.URL)
case "/image.png":
w.Header().Set("Content-Type", "image/png")
w.Write([]byte{1, 2, 3})
}
}))
defer website.Close()
websiteRootURL := website.URL + "/"
websiteImageURL := website.URL + "/image.png"
priv := genNostrKey()
event := &nostr.Event{
CreatedAt: time.Now(),
Kind: nostr.KindTextNote,
Content: fmt.Sprintf("link to an html page with image: %s", websiteRootURL),
PubKey: nostrPubKey(priv),
}
if err := event.Sign(priv); err != nil {
t.Fatal(err)
}
trelay := ServeSingleEvent(t, event)
defer trelay.Close()
cache := DirCache{Root: t.TempDir()}
noxer := Noxer{
Cache: cache,
MaxFileSize: 1024,
KnownRelays: []string{"127.0.0.1"},
IdleRelayTimeout: time.Minute,
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
meta, err := noxer.FetchLinkMeta(ctx, event.ID, trelay.URL, websiteRootURL)
if err != nil {
t.Fatalf("FetchLinkMeta(%s): %v", websiteRootURL, err)
}
var cachedMeta LinkMeta
if err := cache.GetJSON(ctx, MakeCacheKey(websiteRootURL, CacheKeyURLPreview), &cachedMeta); err != nil {
t.Fatalf("cache.getjson: %v", err)
}
compareLinkMeta(t, meta, &cachedMeta)
ds, err := noxer.StreamLinkData(ctx, event.ID, trelay.URL, websiteImageURL)
if err != nil {
t.Fatalf("StreamLinkData(%s): %v", websiteImageURL, err)
}
checkDataStream(t, ds, "image/png", []byte{1, 2, 3})
checkCachedDataFile(t, cache, websiteImageURL, []byte{1, 2, 3})
}
func checkDataStream(t *testing.T, ds *DataStream, ctype string, contents []byte) {
t.Helper()
if err := iotest.TestReader(ds, contents); err != nil {
t.Errorf("data stream reader: %v", err)
}
if ds.ContentType != ctype {
t.Errorf("ds.ContentType = %q; want %q", ds.ContentType, ctype)
}
}
func checkCachedDataFile(t *testing.T, cache DirCache, origURL string, contents []byte) {
t.Helper()
cacheKey := MakeCacheKey(origURL, CacheKeyData)
b, err := os.ReadFile(cache.makeFilepath(cacheKey, false))
if err != nil {
t.Errorf("cache file read: %v", err)
}
if !bytes.Equal(b, contents) {
t.Errorf("cached bytes = %q; want %q", b, contents)
}
}
func compareLinkMeta(t *testing.T, actual, expected *LinkMeta) {
t.Helper()
if actual.Type != expected.Type {
t.Errorf("actual.Type = %q; want %q", actual.Type, expected.Type)
}
if actual.Title != expected.Title {
t.Errorf("actual.Title = %q; want %q", actual.Title, expected.Title)
}
if actual.Description != expected.Description {
t.Errorf("actual.Description = %q; want %q", actual.Description, expected.Description)
}
if len(actual.ImageURLs) != 1 || actual.ImageURLs[0] != expected.ImageURLs[0] {
t.Errorf("actual.ImageURLs = %q; want %q", actual.ImageURLs, expected.ImageURLs)
}
}
func genNostrKey() string {
k := nostr.GeneratePrivateKey()
if k == "" {
panic("nostr.GeneratePrivateKey returned empty string")
}
return k
}
func nostrPubKey(priv string) string {
pub, err := nostr.GetPublicKey(priv)
if err != nil {
panic(err.Error())
}
return pub
}
type FakeNostrRelay struct {
Event *nostr.Event
URL string
HTTPServer *httptest.Server
Mu sync.Mutex
Subs map[string]bool // id => true if still active; false for unsub'ed IDs
}
func (nr *FakeNostrRelay) Close() {
nr.HTTPServer.Close()
}
func (nr *FakeNostrRelay) OpenSubs() []string {
nr.Mu.Lock()
defer nr.Mu.Unlock()
var a []string
for k, open := range nr.Subs {
if open {
a = append(a, k)
}
}
return a
}
func nostrHandler(t *testing.T, nr *FakeNostrRelay) func(*websocket.Conn) {
return func(conn *websocket.Conn) {
for {
var req [3]any
if err := websocket.JSON.Receive(conn, &req); err != nil {
conn.Close()
return
}
switch req[0].(string) {
default:
t.Errorf("ws handler req[0]=%q; want REQ or CLOSE", req[0])
conn.Close()
return
case "CLOSE":
nr.Mu.Lock()
defer nr.Mu.Unlock()
nr.Subs[req[1].(string)] = false
return
case "REQ":
subid := req[1].(string)
nr.Mu.Lock()
nr.Subs[subid] = true
nr.Mu.Unlock()
filters := req[2].(map[string]any)
t.Logf("ws handler sub=%q, filters=%s", subid, filters)
if ids := filters["ids"].([]any); len(ids) != 1 || ids[0].(string) != nr.Event.ID {
t.Errorf("ws handler REQ filter ids=%q; want [%q]", ids, []string{nr.Event.ID})
}
if limit := filters["limit"].(float64); math.Abs(limit-1) > 0.00001 {
t.Errorf("ws handler REQ limit=%f; want 1", limit)
}
b, err := json.Marshal(nr.Event)
if err != nil {
t.Errorf("json.Marshal: %v", err)
conn.Close()
return
}
resp := fmt.Sprintf(`["EVENT", %q, %s]`, subid, b)
t.Logf("ws handler resp: %s", resp)
if err := websocket.Message.Send(conn, resp); err != nil {
t.Errorf("ws handler REQ write: %v", err)
}
}
}
}
}
func ServeSingleEvent(t *testing.T, event *nostr.Event) *FakeNostrRelay {
relay := &FakeNostrRelay{
Event: event,
Subs: make(map[string]bool),
}
relay.HTTPServer = httptest.NewServer(&websocket.Server{
Handshake: func(conf *websocket.Config, r *http.Request) error {
t.Logf("new handshake from %s", r.RemoteAddr)
return nil
},
Handler: nostrHandler(t, relay),
})
tsurl, err := url.Parse(relay.HTTPServer.URL)
if err != nil {
panic(err)
}
relay.URL = fmt.Sprintf("ws://%s/", tsurl.Host)
return relay
}

@ -0,0 +1,6 @@
#!/bin/sh
set -e
VERSION=${VERSION:-$(git describe --tags)}
export CGO_ENABLED=0
exec go build -ldflags "-s -w -X main.noxyVersion=$VERSION" -buildmode=pie -trimpath ./cmd/noxy/
Loading…
Cancel
Save