basic implementation of the last api version

the meat is in Noxer struct, in noxy.go.
executable server entry point is in cmd/noxy/main.go.

all request responses are cached using a rudimentary filesystem
based caching. a max storage quota is not implemented yet.

README.md

@@ -123,6 +123,35 @@ now, fetch the preview image. note the change from `/meta` to `/data` endpoint:
 
 [event raw data →](https://nostr.com/e/af30dac1d800acc25b87d0d6d0dd33bddf49e7f356556540a6c7722e3cb363fe)
 
+## development
 
+the binary's entry point is [cmd/noxy/main.go](cmd/noxy/main.go).
+it imports packages in the root of the repo. the actual proxy is implemented
+by `Noxer` in [noxy.go](noxy.go).
 
+running the server locally:
 
+    mkdir cache
+    go run ./cmd/noxy -cachedir $PWD/cache
+
+before sending a patch, make sure the code is passing tests:
+
+    go test -race
+
+files are formatted:
+
+    go fmt ./...
+
+and the go module file is updated:
+
+    go mod tidy
+
+## release
+
+a release binary is built using the following script.
+it produces `noxy` executable in the root of the repo.
+
+    ./tools/release.sh
+
+before making a new release, you'll probably want to create a new git tag.
+the tag is used as the noxy version, also printed when `-V` flag is specified.

cache.go

@@ -0,0 +1,217 @@
+package noxy
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"mime"
+	"os"
+	"path/filepath"
+)
+
+var ErrCacheMiss = errors.New("cache miss")
+
+// DataStream is an io.Reader augmented with a mime type.
+type DataStream struct {
+	ContentType string
+	r           io.Reader
+}
+
+// MimeType reports the mime type as parsed from ds.ContentType, ignoring
+// any errors resulting from parsing optional media parameters.
+func (ds DataStream) MimeType() string {
+	mtype, _, err := mime.ParseMediaType(ds.ContentType)
+	if err != nil && errors.Is(err, mime.ErrInvalidMediaParameter) {
+		return ds.ContentType
+	}
+	return mtype
+}
+
+func (ds DataStream) Read(p []byte) (n int, err error) {
+	return ds.r.Read(p)
+}
+
+func (ds DataStream) Close() error {
+	if closer, ok := ds.r.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
+}
+
+// Cacher is used by Noxer to store and use meta info in JSON format, and stream files.
+type Cacher interface {
+	GetJSON(ctx context.Context, key CacheKey, dst any) error
+	PutJSON(ctx context.Context, key CacheKey, v any) error
+	GetStream(ctx context.Context, key CacheKey) (*DataStream, error)
+	PutStream(ctx context.Context, key CacheKey, mimeType string, r io.Reader) error
+}
+
+// CacheKeyType allows CacheKey to segregate data based on their logical types.
+type CacheKeyType byte
+
+const (
+	_                  CacheKeyType = 1 << iota
+	CacheKeyEvent                   // nostr event links
+	CacheKeyURLPreview              // OG link preview metadata
+	CacheKeyData                    // actual url data
+)
+
+// CacheKey is 33 bytes long, encoding its CacheKeyType at index 32.
+type CacheKey []byte
+
+// MakeCacheKey creates a new cache key based on sha256 of s and the logical data type.
+func MakeCacheKey(s string, typ CacheKeyType) CacheKey {
+	h := sha256.Sum256([]byte(s))
+	return append(h[:], byte(typ))
+}
+
+// String returns a key representation without its CacheKeyType.
+func (k CacheKey) String() string {
+	return hex.EncodeToString(k[:len(k)-1])
+}
+
+// Path result is suitable as a filesystem path for storing a cache entry.
+func (k CacheKey) Path() string {
+	h := hex.EncodeToString(k[:len(k)-1])
+	return filepath.Join(k.Namespace(), h[0:4], h)
+}
+
+// Namespace is a string representation of the key's CacheKeyType.
+func (k CacheKey) Namespace() string {
+	typ := byte(0)
+	if n := len(k); n > 0 {
+		typ = k[n-1]
+	}
+	switch CacheKeyType(typ) {
+	default:
+		return ""
+	case CacheKeyEvent:
+		return "event"
+	case CacheKeyURLPreview:
+		return "preview"
+	case CacheKeyData:
+		return "data"
+	}
+}
+
+// DirCache implements Cacher using regular filesystem operations.
+// it places all data under subdirectories of the Root.
+//
+// TODO: cap at max storage size
+// TODO: max size per key
+type DirCache struct {
+	Root string
+}
+
+const dirCacheMetaSuffix = ".meta.json"
+
+func (d DirCache) makeFilepath(key CacheKey, mkdir bool) string {
+	p := filepath.Join(d.Root, key.Path())
+	if mkdir {
+		os.MkdirAll(filepath.Dir(p), 0700)
+	}
+	return p
+}
+
+func (d DirCache) makeTemp(key CacheKey) string {
+	p := filepath.Join(d.Root, "tmp", key.Path())
+	os.MkdirAll(filepath.Dir(p), 0700)
+	return p
+}
+
+func (d DirCache) GetJSON(ctx context.Context, key CacheKey, dst any) error {
+	b, err := os.ReadFile(d.makeFilepath(key, false))
+	switch {
+	case err != nil && errors.Is(err, os.ErrNotExist):
+		return ErrCacheMiss
+	case err != nil:
+		return err
+	}
+	return json.Unmarshal(b, dst)
+}
+
+func (d DirCache) PutJSON(ctx context.Context, key CacheKey, v any) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(d.makeFilepath(key, true), b, 0600)
+}
+
+func (d DirCache) GetStream(ctx context.Context, key CacheKey) (*DataStream, error) {
+	filepath := d.makeFilepath(key, false)
+	f, err := os.Open(filepath)
+	switch {
+	case err != nil && errors.Is(err, os.ErrNotExist):
+		return nil, ErrCacheMiss
+	case err != nil:
+		return nil, err
+	}
+	mb, err := os.ReadFile(filepath + dirCacheMetaSuffix)
+	if err != nil {
+		f.Close()
+		return nil, err
+	}
+	var ds DataStream
+	if err := json.Unmarshal(mb, &ds); err != nil {
+		f.Close()
+		return nil, err
+	}
+	ds.r = f
+	return &ds, nil
+}
+
+func (d DirCache) PutStream(ctx context.Context, key CacheKey, mimeType string, r io.Reader) error {
+	ds := DataStream{ContentType: mimeType}
+	mb, err := json.Marshal(ds)
+	if err != nil {
+		return err
+	}
+	tmpfile := d.makeTemp(key)
+	tmpmeta := tmpfile + dirCacheMetaSuffix
+	if err := ioutil.WriteFile(tmpmeta, mb, 0600); err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(tmpfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(f, r); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+
+	filepath := d.makeFilepath(key, true)
+	if err := os.Rename(tmpfile, filepath); err != nil {
+		return err
+	}
+	return os.Rename(tmpmeta, filepath+dirCacheMetaSuffix)
+}
+
+// NullCache stores no data.
+var NullCache nullCache
+
+type nullCache struct{}
+
+func (nc nullCache) GetJSON(context.Context, CacheKey, any) error {
+	return ErrCacheMiss
+}
+func (nc nullCache) PutJSON(context.Context, CacheKey, any) error {
+	return nil
+}
+
+func (nc nullCache) GetStream(context.Context, CacheKey) (*DataStream, error) {
+	return nil, ErrCacheMiss
+}
+func (nc nullCache) PutStream(ctx context.Context, k CacheKey, mtype string, r io.Reader) error {
+	_, err := io.Copy(io.Discard, r)
+	return err
+}

cmd/noxy/main.go

@@ -0,0 +1,206 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"git.qcode.ch/nostr/noxy"
+)
+
+var noxyVersion = "dev" // overwritten by linker flags in release build
+
+var (
+	listenAddr       = flag.String("addr", "127.0.0.1:8889", "listen address")
+	cacheDir         = flag.String("cachedir", "/tmp", "absolute cache dir")
+	maxFileSize      = flag.Int64("maxfilesize", 1<<23, "refuse to handle files larger than this, in bytes")
+	idleRelayTimeout = flag.Duration("idlerelaytimeout", 10*time.Minute, "remove relay connections after idling this long")
+	showVersion      = flag.Bool("V", false, "print version and exit")
+
+	// the -relay flag, populated by parseRelayFlag.
+	// set to defaultKnownRelays if empty.
+	knownRelays        []string
+	defaultKnownRelays = []string{
+		"nostr.x1ddos.ch",
+		// from https://nostr.info/relays/
+		"expensive-relay.fiatjaf.com",
+		"nostr-pub.semisol.dev",
+		"nostr-pub.wellorder.net",
+		"nostr-relay-dev.wlvs.space",
+		"nostr-relay.wlvs.space",
+		"nostr-verified.wellorder.net",
+		"nostr.bitcoiner.social",
+		"nostr.delo.software",
+		"nostr.ono.re",
+		"nostr.onsats.org",
+		"nostr.openchain.fr",
+		"nostr.oxtr.dev",
+		"nostr.rdfriedl.com",
+		"nostr.semisol.dev",
+		"nostr.zaprite.io",
+		"relay.cynsar.foundation",
+		"relay.damus.io",
+		"relay.farscapian.com",
+		"relay.minds.com",
+		"relay.nostr.info",
+		"relay.oldcity-bitcoiners.info",
+		"relay.sovereign-stack.org",
+	}
+)
+
+func init() {
+	flag.Func("relay", "a comma separated nostr relays noxy is allowed to connect to", parseRelayFlag)
+	sort.Strings(defaultKnownRelays)
+}
+
+func parseRelayFlag(v string) error {
+	for _, s := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' }) {
+		s = strings.TrimSpace(s)
+		if s == "" {
+			continue
+		}
+		u, err := url.Parse(s)
+		if err != nil {
+			return fmt.Errorf("invalid relay URL %s: %w", s, err)
+		}
+		host := u.Hostname()
+		if host == "" {
+			return fmt.Errorf("invalid relay URL %s: no hostname", s)
+		}
+		knownRelays = append(knownRelays, host)
+	}
+	return nil
+}
+
+func usage() {
+	w := flag.CommandLine.Output()
+	fmt.Fprintf(w, "usage of %s:\n", os.Args[0])
+	flag.PrintDefaults()
+	fmt.Fprintln(w, "\nthe -relay flag may be specified multiple times.")
+	fmt.Fprintf(w, "its default value is the following list:\n\n")
+	fmt.Fprintln(w, strings.Join(defaultKnownRelays, "\n"))
+}
+
+// set up in main and used by handleXxx HTTP server handlers.
+var noxer *noxy.Noxer
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if *showVersion {
+		fmt.Println(noxyVersion)
+		return
+	}
+	if len(knownRelays) == 0 {
+		knownRelays = defaultKnownRelays
+	}
+	sort.Strings(knownRelays)
+	if !filepath.IsAbs(*cacheDir) {
+		log.Fatal("cache dir must be absolute path")
+	}
+
+	noxer = &noxy.Noxer{
+		MaxFileSize:      *maxFileSize,
+		IdleRelayTimeout: *idleRelayTimeout,
+		KnownRelays:      knownRelays,
+		Cache:            noxy.DirCache{Root: *cacheDir},
+		HTTPClient: &http.Client{Transport: &http.Transport{
+			MaxIdleConns:          100,
+			MaxConnsPerHost:       2,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		}},
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle("/", http.HandlerFunc(handleRoot))
+	mux.Handle("/meta", http.HandlerFunc(handleMeta))
+	mux.Handle("/data", http.HandlerFunc(handleData))
+
+	log.Printf("listening on %s", *listenAddr)
+	log.Printf("known relays: %s", strings.Join(knownRelays, ", "))
+	http.ListenAndServe(*listenAddr, logHandler(mux))
+}
+
+// handles requests to /
+func handleRoot(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintf(w, "this is noxy version %s\nhttps://git.qcode.ch/nostr/noxy\n", noxyVersion)
+}
+
+// handles requests to /meta
+func handleMeta(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+	eventID := r.FormValue("id")
+	relayURL := r.FormValue("relay")
+	linkURL := r.FormValue("url")
+	meta, err := noxer.FetchLinkMeta(r.Context(), eventID, relayURL, linkURL)
+	if err != nil {
+		writeError(w, err)
+		return
+	}
+	res := struct {
+		Type   string   `json:"type"`
+		Title  string   `json:"title"`
+		Descr  string   `json:"descr"`
+		Images []string `json:"images"`
+	}{
+		Type:   meta.Type,
+		Title:  meta.Title,
+		Descr:  meta.Description,
+		Images: meta.ImageURLs,
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(res)
+}
+
+// handles requests to /data
+func handleData(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+	eventID := r.FormValue("id")
+	relayURL := r.FormValue("relay")
+	linkURL := r.FormValue("url")
+	ds, err := noxer.StreamLinkData(r.Context(), eventID, relayURL, linkURL)
+	if err != nil {
+		writeError(w, err)
+		return
+	}
+	defer ds.Close()
+	w.Header().Set("Content-Type", ds.ContentType)
+	io.Copy(w, ds)
+}
+
+func writeError(w http.ResponseWriter, err error) {
+	log.Printf("ERROR: %v", err)
+	w.Header().Set("Content-Type", "text/plain")
+	switch {
+	default:
+		w.WriteHeader(http.StatusBadRequest)
+	case errors.Is(err, noxy.ErrNotFound):
+		w.WriteHeader(http.StatusNotFound)
+	}
+	fmt.Fprint(w, err.Error())
+}
+
+func logHandler(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("%s %s", r.Method, r.RequestURI)
+		h.ServeHTTP(w, r)
+	})
+}

go.mod

@@ -0,0 +1,21 @@
+module git.qcode.ch/nostr/noxy
+
+go 1.18
+
+require (
+	github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a
+	github.com/nbd-wtf/go-nostr v0.8.1
+	golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
+	mvdan.cc/xurls/v2 v2.4.0
+)
+
+require (
+	github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
+	github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/valyala/fastjson v1.6.3 // indirect
+	golang.org/x/exp v0.0.0-20221106115401-f9659909a136 // indirect
+)

go.sum

@@ -0,0 +1,38 @@
+github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8 h1:Xa6tp8DPDhdV+k23uiTC/GrAYOe4IdyJVKtob4KW3GA=
+github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20220414055132-a37292614db8/go.mod h1:ihkm1viTbO/LOsgdGoFPBSvzqvx7ibvkMzYp3CgtHik=
+github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
+github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
+github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
+github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a h1:etIrTD8BQqzColk9nKRusM9um5+1q0iOEJLqfBMIK64=
+github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a/go.mod h1:emQhSYTXqB0xxjLITTw4EaWZ+8IIQYw+kx9GqNUKdLg=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/nbd-wtf/go-nostr v0.8.1 h1:DCbLiF1r3xHKBQA1Noz+97ra/B9AcftTh9w+syg3KzM=
+github.com/nbd-wtf/go-nostr v0.8.1/go.mod h1:IIT/16QZ/nzi5cgQFU2WJrezYPNRi0iNgiitYMiu8UQ=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
+github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
+github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
+golang.org/x/exp v0.0.0-20221106115401-f9659909a136 h1:Fq7F/w7MAa1KJ5bt2aJ62ihqp9HDcRuyILskkpIAurw=
+golang.org/x/exp v0.0.0-20221106115401-f9659909a136/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
+golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc=
+mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg=

io.go

@@ -0,0 +1,53 @@
+package noxy
+
+import (
+	"bufio"
+	"errors"
+	"io"
+)
+
+var ErrSizeLimitExceeded = errors.New("size limit exceeded")
+
+// HardLimitReader reads from r up to max bytes.
+// it returns ErrSizeLimitExceeded if the number of read bytes is equal or exceeds max.
+func HardLimitReader(r io.Reader, max int64) io.Reader {
+	return &hardLimitReader{r, max}
+}
+
+type hardLimitReader struct {
+	r io.Reader
+	n int64 // remaining bytes
+}
+
+func (l *hardLimitReader) Read(p []byte) (n int, err error) {
+	if l.n <= 0 {
+		return 0, ErrSizeLimitExceeded
+	}
+	if int64(len(p)) > l.n {
+		p = p[0:l.n]
+	}
+	n, err = l.r.Read(p)
+	l.n -= int64(n)
+	return
+}
+
+// SinkTeeReader stops writing to w at the first encountered write error
+// but continues to propagate reads. the underlying writer is always buffered.
+func SinkTeeReader(r io.Reader, w io.Writer) io.Reader {
+	return &sinkTeeReader{r: r, w: bufio.NewWriter(w)}
+}
+
+type sinkTeeReader struct {
+	r    io.Reader
+	w    *bufio.Writer
+	werr error
+}
+
+func (st *sinkTeeReader) Read(p []byte) (n int, err error) {
+	n, err = st.r.Read(p)
+	if n > 0 && st.werr == nil {
+		_, st.werr = st.w.Write(p[:n])
+		st.werr = st.w.Flush()
+	}
+	return
+}

io_test.go

@@ -0,0 +1,37 @@
+package noxy
+
+import (
+	"bytes"
+	"testing"
+	"testing/iotest"
+)
+
+// ErrorWriter always returns Err on Write calls.
+type ErrorWriter struct {
+	Err error
+}
+
+func (ew ErrorWriter) Write([]byte) (int, error) {
+	return 0, ew.Err
+}
+
+func TestSinkTeeReader(t *testing.T) {
+	const text = "hello"
+	var w bytes.Buffer
+	tee := SinkTeeReader(bytes.NewBufferString(text), &w)
+
+	if err := iotest.TestReader(tee, []byte(text)); err != nil {
+		t.Errorf("tee reader: %v", err)
+	}
+	if v := string(w.Bytes()); v != text {
+		t.Errorf("b2 = %q; want %q", v, text)
+	}
+}
+
+func TestSinkTeeReaderErrWriter(t *testing.T) {
+	const text = "hello"
+	tee := SinkTeeReader(bytes.NewBufferString(text), ErrorWriter{iotest.ErrTimeout})
+	if err := iotest.TestReader(tee, []byte(text)); err != nil {
+		t.Errorf("tee reader: %v", err)
+	}
+}

noxy.go

@@ -0,0 +1,593 @@
+package noxy
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"path"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/dyatlov/go-opengraph/opengraph"
+	nostr "github.com/nbd-wtf/go-nostr"
+	xurls "mvdan.cc/xurls/v2"
+)
+
+var (
+	ErrNotFound             = errors.New("event or resource not found")
+	ErrUnsupportedEventKind = errors.New("unsupported event kind")
+	ErrUnsupportedMimeType  = errors.New("unsupported link mime type")
+	ErrUnsupportedRelay     = errors.New("unsupported relay")
+)
+
+// LinkMeta contains metadata info about a URL.
+// it is typically assembled from OGP (https://ogp.me) by Noxer.FetchLinkMeta.
+type LinkMeta struct {
+	Type        string   // og:type
+	Title       string   // og:title
+	Description string   // og:description
+	ImageURLs   []string // og:image:secure_url or og:image:url
+}
+
+// Noxer can proxy link preview info and data streams.
+// See FetchLinkMeta and StreamLinkData for details.
+//
+// while the only required field is Cache, a zero value of KnownRelays
+// makes Noxer refuse to proxy any URLs.
+type Noxer struct {
+	// Cache is used to store both link preview meta info and
+	// data streamed to clients. it must be non-nil for Noxer to be usable.
+	Cache Cacher
+
+	// Noxer refuses to work with web pages and data streams larger than this value.
+	MaxFileSize int64 // defaults to 1Mb
+	// how long to keep an open connection to a relay without any activity.
+	// an activity is any cache-miss call to FetchLinkMeta or StreamLinkData.
+	// connections to relays are used to verify whether a link is part of
+	// an event contents. see aforementioned methods for more details.
+	IdleRelayTimeout time.Duration // defaults to 1min
+	// Noxer connects only to those relays hostnames of which are specified here.
+	// in other words, slice elements are only hostname parts of relay URLs.
+	// KnownRelays must be sorted in ascending order.
+	KnownRelays []string
+
+	// HTTPClient is used to make HTTP connections when fetching link preview
+	// info and data streaming. when nil, http.DefaultClient is used.
+	HTTPClient *http.Client
+
+	// clients keeps track of nostr relay connections to clean them up
+	// and remove idle after IdleRelayTimeout.
+	clientsMu    sync.Mutex
+	clients      map[string]*relayClient
+	cleanupTimer *time.Timer
+
+	// slurpers keep track of ongoing HTTP requests, both link preview
+	// meta info and data streams.
+	slurpersMu sync.Mutex
+	slurpers   map[string]chan struct{}
+}
+
+// relayClient wraps nostr.Relay with an additional timestamp
+// indicating last use of the relay to keep track of all active relay
+// connections and remove idle.
+//
+// lastUsed is updated every time Noxer.fetchNostrEvent is called.
+type relayClient struct {
+	relay    *nostr.Relay
+	lastUsed time.Time
+}
+
+// FetchLinkMeta requests the web page at link URL, parses it as HTML and returns
+// metadata found in the contents. It refuses to parse remote responses with
+// content-type other than text/html.
+//
+// link URL must be found in content field of the nostr event posted to the
+// specified relay. FetchLinkMeta connects to the nostr relay at relayURL
+// and sends a filter'ed request with ids field set to eventID.
+// the received event contents are "grepped" for the value of link as is.
+//
+// relayURL's hostname must be an element of x.KnownRelays.
+// remote must respond with HTTP 200 OK to the link URL.
+//
+// successfully parsed link URLs are cached using Cacher.PutJSON. so, subsequent
+// calls should not hit the remote server again unless x.Cache fails.
+// concurrent requests are suspended until the context or first call is done.
+func (x *Noxer) FetchLinkMeta(ctx context.Context, eventID, relayURL, link string) (*LinkMeta, error) {
+	if err := x.verifyEventLink(ctx, eventID, relayURL, link, verifyNoMeta); err != nil {
+		return nil, fmt.Errorf("verifyEventLink: %w", err)
+	}
+	return x.slurpLinkMeta(ctx, link)
+}
+
+func (x *Noxer) slurpLinkMeta(ctx context.Context, link string) (*LinkMeta, error) {
+	// use cache here instead of directly in FetchLinkMeta to avoid
+	// hitting remotes in x.verifyEventLink as much as possible.
+	cacheKey := MakeCacheKey(link, CacheKeyURLPreview)
+	var meta LinkMeta
+	cacheErr := x.Cache.GetJSON(ctx, cacheKey, &meta)
+	if cacheErr == nil {
+		return &meta, nil
+	}
+
+	log.Printf("cache.getjson %s(%s): %v", link, cacheKey, cacheErr)
+	ds, err := x.detachedSlurpData(ctx, link)
+	if err != nil {
+		return nil, fmt.Errorf("detachedSlurpData: %w", err)
+	}
+	defer ds.Close()
+	if mtype := ds.MimeType(); mtype != "text/html" {
+		return nil, fmt.Errorf("%w: received %q, want text/html", ErrUnsupportedMimeType, mtype)
+	}
+	res, err := parseLinkMeta(ds)
+	if err != nil {
+		return nil, fmt.Errorf("parseLinkMeta: %w", err)
+	}
+	if err := x.Cache.PutJSON(ctx, cacheKey, res); err != nil {
+		log.Printf("cache.putjson %s(%s): %v", link, cacheKey, err)
+	}
+	return res, nil
+}
+
+// StreamLinkData opens an HTTP connection to link and streams the response back.
+// while doing so, it also caches the reponse bytes using Cache.PutStream. so,
+// subsequent calls should not hit the remote link again unless x.Cache fails.
+//
+// link URL must be found in "content" field of the nostr event posted to the
+// specified relay. StreamLinkData connects to the nostr relay at relayURL
+// and sends a filter'ed request with ids field set to eventID.
+// for event kinds 1 (text note) and 42 (channel message), the event contents
+// are simply "grepped" for the value of link as is.
+// for event kinds 0 (set metadata), 40 (create channel) and 41 (set channel
+// metadata) the link is checked against "picture" field.
+//
+// additionally, link URL may be one of LinkMeta.ImageURLs as returned by
+// x.FetchLinkMeta to a call with the same eventID.
+//
+// relayURL's hostname must be an element of x.KnownRelays.
+// remote must respond with HTTP 200 OK to the link URL.
+//
+// callers must close DataStream.
+// concurrent requests are suspended until the context or first call is done.
+func (x *Noxer) StreamLinkData(ctx context.Context, eventID, relayURL, link string) (*DataStream, error) {
+	if err := x.verifyEventLink(ctx, eventID, relayURL, link, verifyExpandMeta); err != nil {
+		return nil, err
+	}
+	cacheKey := MakeCacheKey(link, CacheKeyData)
+	ds, err := x.Cache.GetStream(ctx, cacheKey)
+	if err != nil {
+		log.Printf("cache.getstream %s(%s): %v", link, cacheKey, err)
+		ds, err = x.detachedSlurpData(ctx, link)
+	}
+	return ds, err
+}
+
+// detachedSlurpData always finishes data streaming from remote url, event if
+// the returned DataStream is closed prematurely, to cache the bytes for subsequent calls.
+func (x *Noxer) detachedSlurpData(ctx context.Context, url string) (*DataStream, error) {
+	// check whether there's an ongoing stream. if so, wait and use cache or fail.
+	cacheKey := MakeCacheKey(url, CacheKeyData)
+	cacheKeyStr := cacheKey.Path()
+	x.slurpersMu.Lock()
+	slurpCh, found := x.slurpers[cacheKeyStr]
+	if found {
+		// a previous call is already streaming.
+		// wait 'till they're done, because the stream is non-seekable,
+		// then get it from cache or fail.
+		x.slurpersMu.Unlock()
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-slurpCh:
+			return x.Cache.GetStream(ctx, cacheKey)
+		}
+	} else {
+		// wouldn't need this branch if close(slurpCh) was done after x.slurpersMu.Lock()
+		// in the goroutine below.
+		// but it's so easy to miss in future code changes that i don't want to risk it:
+		// not a big deal to check the cache one more time.
+		// reconsider if performance here becomes a concern.
+		ds, err := x.Cache.GetStream(ctx, cacheKey)
+		if err == nil {
+			x.slurpersMu.Unlock()
+			return ds, nil
+		}
+	}
+
+	// no other goroutine is streaming; do it now and make others wait on slurpCh.
+	slurpCh = x.makeSlurperChan(cacheKeyStr)
+	x.slurpersMu.Unlock()
+
+	// assuming 1min is enough to download a file.
+	// this may be too short for large values of x.MaxFileSize.
+	// TODO: compute ctx based on x.MaxFileSize?
+	ctx, cancelHTTP := context.WithTimeout(context.Background(), time.Minute)
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		cancelHTTP()
+		return nil, err
+	}
+	resp, err := x.httpClient().Do(req)
+	if err != nil {
+		cancelHTTP()
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		cancelHTTP()
+		if resp.StatusCode == http.StatusNotFound {
+			return nil, ErrNotFound
+		}
+		return nil, fmt.Errorf("bad HTTP response %s: %s", url, resp.Status)
+	}
+	ctype := resp.Header.Get("Content-Type")
+	if ctype == "" {
+		// TODO: sniff using mime magic bytes?
+		ctype = "application/octet-stream"
+	}
+	// rout is returned to the caller, wout is tee'ed from resp.Body.
+	// if the caller closes rout, tee'ing to wout also stops.
+	rout, wout := io.Pipe()
+	go func() {
+		defer func() {
+			resp.Body.Close()
+			wout.Close()
+			cancelHTTP()
+			close(slurpCh)
+			x.slurpersMu.Lock()
+			delete(x.slurpers, cacheKeyStr)
+			x.slurpersMu.Unlock()
+		}()
+		// the std io.TeeReader wouldn't work since it reports errors on reads
+		// from tee as soon as writes to wout fail which is the case if the caller
+		// closes rout.
+		tee := SinkTeeReader(HardLimitReader(resp.Body, x.maxFileSize()), wout)
+		if err := x.Cache.PutStream(ctx, cacheKey, ctype, tee); err != nil {
+			log.Printf("cache.putstream %s: %v", cacheKey, err)
+			// TODO: don't close; io.copy(wout, resp.body) here on cache failures?
+		}
+	}()
+	return &DataStream{ContentType: ctype, r: rout}, nil
+}
+
+// expandMeta arg values for verifyEventLink
+const (
+	verifyExpandMeta = true
+	verifyNoMeta     = false
+)
+
+// verifyEventLink checks whether link URL is in a nostr event's content,
+// or one of OGP link preview URLs if expandMeta is true.
+func (x *Noxer) verifyEventLink(ctx context.Context, eventID, relayURL, link string, expandMeta bool) error {
+	if !x.whitelistedRelay(relayURL) {
+		return ErrUnsupportedRelay
+	}
+	eventURLs, err := x.fetchEventURLs(ctx, eventID, relayURL)
+	if err != nil {
+		return err
+	}
+	log.Printf("fetched event URLs: %q", eventURLs)
+	for _, u := range eventURLs {
+		if u == link {
+			return nil
+		}
+	}
+	if !expandMeta {
+		return ErrNotFound
+	}
+
+	// link not found in the event text/json.
+	// check URLs in OGP metadata for each suitable link found in the event.
+	for _, urlStr := range eventURLs {
+		u, err := url.Parse(urlStr)
+		if err != nil {
+			continue // invalid url
+		}
+		if ext := path.Ext(u.Path); ext != "" {
+			if !strings.HasSuffix(ext, "html") && !strings.HasSuffix(ext, "htm") {
+				continue // assume not an html page
+			}
+		}
+		meta, err := x.slurpLinkMeta(ctx, urlStr)
+		if err != nil {
+			log.Printf("verifyEventLink slurpLinkMeta(%s): %v", u, err)
+			continue
+		}
+		for _, imgURL := range meta.ImageURLs {
+			if imgURL == link {
+				return nil
+			}
+		}
+	}
+	return ErrNotFound
+}
+
+// fetchEventURLs returns all URLs found in a nostr event.
+// it assumes the relay URL is already checked to match x.KnownRelays.
+func (x *Noxer) fetchEventURLs(ctx context.Context, eventID, relayURL string) ([]string, error) {
+	// check whether there's an ongoing fetch. if so, wait and use cache or fail.
+	cacheKey := MakeCacheKey(eventID, CacheKeyEvent)
+	cacheKeyStr := cacheKey.Path()
+	x.slurpersMu.Lock()
+	slurpCh, found := x.slurpers[cacheKeyStr]
+	if found {
+		// a previous call is already fetching.
+		// wait 'till they're done, then get it from cache or fail.
+		x.slurpersMu.Unlock()
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-slurpCh:
+			var urls []string
+			err := x.Cache.GetJSON(ctx, cacheKey, &urls)
+			return urls, err
+		}
+	} else {
+		// same reasoning as in detachedSlurpData.
+		// wouldn't need this branch if close(slurpCh) was done after x.slurpersMu.Lock()
+		// in the goroutine below. but it's too easy to miss in future code changes.
+		// checking cache one more time here is most likely insignificant when compared to
+		// opening a websocket to a nostr relay.
+		var urls []string
+		if err := x.Cache.GetJSON(ctx, cacheKey, &urls); err == nil {
+			x.slurpersMu.Unlock()
+			return urls, nil
+		}
+	}
+
+	// no other goroutine is fetching; do it now and make others wait on slurpCh.
+	slurpCh = x.makeSlurperChan(cacheKeyStr)
+	x.slurpersMu.Unlock()
+	defer func() {
+		close(slurpCh)
+		x.slurpersMu.Lock()
+		delete(x.slurpers, cacheKeyStr)
+		x.slurpersMu.Unlock()
+	}()
+
+	event, err := x.fetchNostrEvent(ctx, eventID, relayURL)
+	if err != nil {
+		return nil, err
+	}
+	var eventURLs []string
+	switch event.Kind {
+	default:
+		return nil, ErrUnsupportedEventKind
+	case nostr.KindTextNote, nostr.KindChannelMessage:
+		eventURLs = extractAcceptableURLs(event.Content)
+	case nostr.KindSetMetadata, nostr.KindChannelCreation, nostr.KindChannelMetadata:
+		var p struct{ Picture string }
+		if err := json.Unmarshal([]byte(event.Content), &p); err != nil {
+			return nil, err
+		}
+		if validURL(p.Picture) {
+			eventURLs = append(eventURLs, p.Picture)
+		}
+	}
+
+	if err := x.Cache.PutJSON(ctx, cacheKey, eventURLs); err != nil {
+		log.Printf("cache.putjson %s: %v", cacheKey, err)
+	}
+	return eventURLs, nil
+}
+
+// assuming relay is whitelisted
+func (x *Noxer) fetchNostrEvent(ctx context.Context, eventID, relayURL string) (*nostr.Event, error) {
+	relay, err := x.relayConn(ctx, relayURL)
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		event    *nostr.Event
+		fetchErr error
+	)
+	// assuming 10sec is more than enough for a simple filter'ed sub with a single
+	// event ID.
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		f := nostr.Filter{IDs: []string{eventID}, Limit: 1}
+		sub := relay.Subscribe(nostr.Filters{f})
+		defer sub.Unsub()
+		select {
+		case e := <-sub.Events:
+			// e.CheckSignature() is already done by the client
+			event = &e
+		case <-ctx.Done():
+			fetchErr = ctx.Err()
+		}
+	}()
+
+	select {
+	case <-done:
+		return event, fetchErr
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+// connect to a nostr relay at relayURL or reuse an existing conn.
+// it blocks all other callers.
+func (x *Noxer) relayConn(ctx context.Context, relayURL string) (*nostr.Relay, error) {
+	// check existing conn and reuse if found
+	relayURL = nostr.NormalizeURL(relayURL)
+	x.clientsMu.Lock()
+	defer x.clientsMu.Unlock()
+	if cl, ok := x.clients[relayURL]; ok {
+		// "touch" the last used to let cleanup timer know we aren't idling
+		cl.lastUsed = time.Now()
+		return cl.relay, nil
+	}
+
+	// none found. make a new conn.
+	var (
+		relay   *nostr.Relay
+		connErr error
+	)
+	// assuming 10sec is more than enough to connect to a websocket.
+	connCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() {
+		// TODO: send a patch upstream for a nostr.RelayConnectContext(ctx, url)
+		relay, connErr = nostr.RelayConnect(relayURL)
+		close(done)
+	}()
+	select {
+	case <-connCtx.Done():
+		// unfortunately, this leaves the above goroutine hanging, and will keep
+		// piling up for non-responsive relays.
+		// can be solved with a nostr.RelayConnectContext.
+		return nil, connCtx.Err()
+	case <-done:
+		if connErr != nil {
+			return nil, connErr
+		}
+	}
+	if x.clients == nil {
+		x.clients = make(map[string]*relayClient)
+	}
+	x.clients[relayURL] = &relayClient{
+		relay:    relay,
+		lastUsed: time.Now(),
+	}
+	// a self-cleanup goroutine to delete ourselves if relay reports conn errors.
+	go func() {
+		err := <-relay.ConnectionError
+		log.Printf("%s: closing due to: %v", relayURL, err)
+		x.clientsMu.Lock()
+		defer x.clientsMu.Unlock()
+		relay.Close()
+		delete(x.clients, relayURL)
+	}()
+	if x.cleanupTimer == nil {
+		x.cleanupTimer = time.AfterFunc(x.idleRelayTimeout(), x.cleanupRelayConn)
+	}
+	return relay, nil
+}
+
+// close and delete nostr relay connections idling for more than x.idleRelayTimeout().
+func (x *Noxer) cleanupRelayConn() {
+	x.clientsMu.Lock()
+	defer x.clientsMu.Unlock()
+	for url, cl := range x.clients {
+		if time.Since(cl.lastUsed) > x.idleRelayTimeout() {
+			log.Printf("closing idle conn to %s", url)
+			cl.relay.Close()
+			delete(x.clients, url)
+		}
+	}
+	if len(x.clients) > 0 {
+		x.cleanupTimer = time.AfterFunc(time.Minute, x.cleanupRelayConn)
+	} else {
+		x.cleanupTimer = nil
+	}
+}
+
+// assumes x.slurpersMu is handled by the caller.
+func (x *Noxer) makeSlurperChan(k string) chan struct{} {
+	if x.slurpers == nil {
+		x.slurpers = make(map[string]chan struct{})
+	}
+	ch := make(chan struct{})
+	x.slurpers[k] = ch
+	return ch
+}
+
+func (x *Noxer) httpClient() *http.Client {
+	if x.HTTPClient == nil {
+		return http.DefaultClient
+	}
+	return x.HTTPClient
+}
+
+func (x *Noxer) idleRelayTimeout() time.Duration {
+	if x.IdleRelayTimeout == 0 {
+		return time.Minute
+	}
+	return x.IdleRelayTimeout
+}
+
+func (x *Noxer) maxFileSize() int64 {
+	if x.MaxFileSize == 0 {
+		return 1 << 20 // 1Mb
+	}
+	return x.MaxFileSize
+}
+
+// whitelistedRelay reports whether a nostr relay at urlStr is in x.KnownRelays.
+// it expects x.KnownRelays to be sorted in lexical order.
+//
+// only hostname of urlStr is checked against x.KnownRelays.
+func (x *Noxer) whitelistedRelay(urlStr string) bool {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return false
+	}
+	host := u.Hostname()
+	i := sort.SearchStrings(x.KnownRelays, host)
+	return i < len(x.KnownRelays) && x.KnownRelays[i] == host
+}
+
+// TODO: use oEmbed if OGP fails?
+func parseLinkMeta(r io.Reader) (*LinkMeta, error) {
+	og := opengraph.NewOpenGraph()
+	if err := og.ProcessHTML(r); err != nil {
+		return nil, err
+	}
+	if len(og.Images) == 0 {
+		return nil, ErrNotFound
+	}
+	meta := &LinkMeta{
+		Type:        og.Type,
+		Title:       og.Title,
+		Description: og.Description,
+		ImageURLs:   make([]string, 0, len(og.Images)),
+	}
+	for _, img := range og.Images {
+		u := img.SecureURL
+		if u == "" {
+			u = img.URL
+		}
+		if u == "" {
+			continue
+		}
+		meta.ImageURLs = append(meta.ImageURLs, u)
+	}
+	return meta, nil
+}
+
+// TODO: patch to extract only host/ip; no emails and such
+var urlRegexp = xurls.Relaxed()
+
+func extractAcceptableURLs(text string) []string {
+	var urls []string
+	for _, a := range urlRegexp.FindAllString(text, -1) {
+		if validURL(a) {
+			urls = append(urls, a)
+		}
+	}
+	return urls
+}
+
+func validURL(urlStr string) bool {
+	if urlStr == "" {
+		return false
+	}
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return false
+	}
+	if u.Hostname() == "" {
+		return false
+	}
+	return u.Scheme == "" || u.Scheme == "http" || u.Scheme == "https"
+}

noxy_test.go

@@ -0,0 +1,414 @@
+package noxy
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"sync"
+	"testing"
+	"testing/iotest"
+	"time"
+
+	nostr "github.com/nbd-wtf/go-nostr"
+	"golang.org/x/net/websocket"
+)
+
+func TestDetachedSlurpDataCacheMiss(t *testing.T) {
+	const contents = "text file"
+	const ctype = "text/plain;charset=utf-8"
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", ctype)
+		w.Write([]byte(contents))
+	}))
+	defer ts.Close()
+	var testURL = ts.URL + "/"
+
+	cache := DirCache{Root: t.TempDir()}
+	noxer := Noxer{Cache: cache, MaxFileSize: 1024}
+
+	for i := 1; i <= 2; i++ {
+		t.Run(fmt.Sprintf("slurp %d", i), func(t *testing.T) {
+			bgCtx := context.Background()
+			canceledCtx, cancel := context.WithCancel(bgCtx)
+			cancel() // slurp must run on a separate context
+			ds, err := noxer.detachedSlurpData(canceledCtx, testURL)
+			if err != nil {
+				t.Fatalf("noxer.detachedSlurpData: %v", err)
+			}
+			checkDataStream(t, ds, ctype, []byte(contents))
+
+			checkCachedDataFile(t, cache, testURL, []byte(contents))
+			cacheKey := MakeCacheKey(testURL, CacheKeyData)
+			cachedDS, err := cache.GetStream(bgCtx, cacheKey)
+			if err != nil {
+				t.Fatalf("cache.GetStream: %v", err)
+			}
+			checkDataStream(t, cachedDS, ctype, []byte(contents))
+
+			noxer.slurpersMu.Lock()
+			defer noxer.slurpersMu.Unlock()
+			if len(noxer.slurpers) > 0 {
+				t.Error("x.slurpers is not empty")
+			}
+		})
+	}
+}
+
+func TestDetachedSlurpDataClosedReader(t *testing.T) {
+	const ctype = "text/plain;charset=utf-8"
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", ctype)
+		w.Write([]byte("foo"))
+		time.Sleep(time.Second)
+		w.Write([]byte("bar"))
+	}))
+	defer ts.Close()
+	var testURL = ts.URL + "/"
+
+	cache := DirCache{Root: t.TempDir()}
+	noxer := Noxer{Cache: cache, MaxFileSize: 1024}
+
+	ctx := context.Background()
+	ds1, err := noxer.detachedSlurpData(ctx, testURL)
+	if err != nil {
+		t.Fatalf("noxer.detachedSlurpData 1: %v", err)
+	}
+	ds1.r.(io.Closer).Close()
+
+	cacheKey := MakeCacheKey(testURL, CacheKeyData)
+	noxer.slurpersMu.Lock()
+	ch := noxer.slurpers[cacheKey.Path()]
+	noxer.slurpersMu.Unlock()
+	select {
+	case <-time.After(3 * time.Second):
+		t.Fatal("slurp took too long")
+	case <-ch:
+	}
+
+	ds2, err := cache.GetStream(ctx, cacheKey)
+	if err != nil {
+		t.Fatalf("cache.GetStream: %v", err)
+	}
+	checkDataStream(t, ds2, ctype, []byte("foobar"))
+}
+
+func TestSlurpLinkMeta(t *testing.T) {
+	var count int
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if count > 0 {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		count += 1
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprintln(w, `<html><head>
+		<meta property="og:type" content="article" />
+		<meta property="og:title" content="test title" />
+		<meta property="og:description" content="test descr" />
+		<meta property="og:image" content="http://unused:0/image.png" />
+		</head></html>`)
+	}))
+	defer ts.Close()
+	var testURL = ts.URL + "/"
+
+	cache := DirCache{Root: t.TempDir()}
+	noxer := Noxer{Cache: cache, MaxFileSize: 1024}
+	meta1, err := noxer.slurpLinkMeta(context.Background(), testURL)
+	if err != nil {
+		t.Fatalf("slurpLinkMeta 1: %v", err)
+	}
+	wantMeta := &LinkMeta{
+		Type:        "article",
+		Title:       "test title",
+		Description: "test descr",
+		ImageURLs:   []string{"http://unused:0/image.png"},
+	}
+	compareLinkMeta(t, meta1, wantMeta)
+
+	// expected to be cached by now
+	meta2, err := noxer.slurpLinkMeta(context.Background(), testURL)
+	if err != nil {
+		t.Fatalf("slurpLinkMeta 2: %v", err)
+	}
+	compareLinkMeta(t, meta2, wantMeta)
+}
+
+func TestSlurpLinkMetaHTTPErr(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer ts.Close()
+	var testURL = ts.URL + "/"
+
+	noxer := Noxer{Cache: NullCache, MaxFileSize: 1024}
+	_, err := noxer.slurpLinkMeta(context.Background(), testURL)
+	if !errors.Is(err, ErrNotFound) {
+		t.Errorf("slurpLinkMeta err=%v; want ErrNotFound", err)
+	}
+}
+
+func TestVerifyEventLinkNoMeta(t *testing.T) {
+	priv := genNostrKey()
+	event := &nostr.Event{
+		CreatedAt: time.Now(),
+		Kind:      nostr.KindTextNote,
+		Content:   "text; http://unused:0/foo and http://unused:0/bar",
+		PubKey:    nostrPubKey(priv),
+	}
+	if err := event.Sign(priv); err != nil {
+		t.Fatal(err)
+	}
+
+	trelay := ServeSingleEvent(t, event)
+	defer trelay.Close()
+	t.Logf("fake relay URL: %s", trelay.URL)
+
+	noxer := Noxer{
+		Cache:            DirCache{Root: t.TempDir()},
+		MaxFileSize:      1024,
+		KnownRelays:      []string{"127.0.0.1"},
+		IdleRelayTimeout: time.Minute,
+	}
+	tt := []struct {
+		url    string
+		wantOK bool
+	}{
+		{"http://unused:0/foo", true},
+		{"http://unused:0/bar", true},
+		{"http://unused:0/", false},
+		{"http://example.org", false},
+	}
+	for _, tc := range tt {
+		t.Run(tc.url, func(t *testing.T) {
+			ctx := context.Background()
+			err := noxer.verifyEventLink(ctx, event.ID, trelay.URL, tc.url, verifyNoMeta)
+			switch {
+			case tc.wantOK && err != nil:
+				t.Errorf("verifyEventLink: %v", err)
+			case !tc.wantOK && err == nil:
+				t.Error("verifyEventLink returned nil error")
+			}
+		})
+	}
+
+	if subs := trelay.OpenSubs(); len(subs) > 0 {
+		t.Errorf("trelay.OpenSubs is not empty: %q", subs)
+	}
+}
+
+func TestFetchMetaAndStreamData(t *testing.T) {
+	var website *httptest.Server
+	website = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		default:
+			w.WriteHeader(http.StatusBadRequest)
+			t.Errorf("%s %s", r.Method, r.URL)
+		case "/":
+			w.Header().Set("Content-Type", "text/html")
+			fmt.Fprintf(w, `<html><head>
+			<meta property="og:image" content="%s/image.png" />
+			</head></html>`, website.URL)
+		case "/image.png":
+			w.Header().Set("Content-Type", "image/png")
+			w.Write([]byte{1, 2, 3})
+		}
+	}))
+	defer website.Close()
+	websiteRootURL := website.URL + "/"
+	websiteImageURL := website.URL + "/image.png"
+
+	priv := genNostrKey()
+	event := &nostr.Event{
+		CreatedAt: time.Now(),
+		Kind:      nostr.KindTextNote,
+		Content:   fmt.Sprintf("link to an html page with image: %s", websiteRootURL),
+		PubKey:    nostrPubKey(priv),
+	}
+	if err := event.Sign(priv); err != nil {
+		t.Fatal(err)
+	}
+	trelay := ServeSingleEvent(t, event)
+	defer trelay.Close()
+
+	cache := DirCache{Root: t.TempDir()}
+	noxer := Noxer{
+		Cache:            cache,
+		MaxFileSize:      1024,
+		KnownRelays:      []string{"127.0.0.1"},
+		IdleRelayTimeout: time.Minute,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	meta, err := noxer.FetchLinkMeta(ctx, event.ID, trelay.URL, websiteRootURL)
+	if err != nil {
+		t.Fatalf("FetchLinkMeta(%s): %v", websiteRootURL, err)
+	}
+	var cachedMeta LinkMeta
+	if err := cache.GetJSON(ctx, MakeCacheKey(websiteRootURL, CacheKeyURLPreview), &cachedMeta); err != nil {
+		t.Fatalf("cache.getjson: %v", err)
+	}
+	compareLinkMeta(t, meta, &cachedMeta)
+
+	ds, err := noxer.StreamLinkData(ctx, event.ID, trelay.URL, websiteImageURL)
+	if err != nil {
+		t.Fatalf("StreamLinkData(%s): %v", websiteImageURL, err)
+	}
+	checkDataStream(t, ds, "image/png", []byte{1, 2, 3})
+	checkCachedDataFile(t, cache, websiteImageURL, []byte{1, 2, 3})
+}
+
+func checkDataStream(t *testing.T, ds *DataStream, ctype string, contents []byte) {
+	t.Helper()
+	if err := iotest.TestReader(ds, contents); err != nil {
+		t.Errorf("data stream reader: %v", err)
+	}
+	if ds.ContentType != ctype {
+		t.Errorf("ds.ContentType = %q; want %q", ds.ContentType, ctype)
+	}
+}
+
+func checkCachedDataFile(t *testing.T, cache DirCache, origURL string, contents []byte) {
+	t.Helper()
+	cacheKey := MakeCacheKey(origURL, CacheKeyData)
+	b, err := os.ReadFile(cache.makeFilepath(cacheKey, false))
+	if err != nil {
+		t.Errorf("cache file read: %v", err)
+	}
+	if !bytes.Equal(b, contents) {
+		t.Errorf("cached bytes = %q; want %q", b, contents)
+	}
+}
+
+func compareLinkMeta(t *testing.T, actual, expected *LinkMeta) {
+	t.Helper()
+	if actual.Type != expected.Type {
+		t.Errorf("actual.Type = %q; want %q", actual.Type, expected.Type)
+	}
+	if actual.Title != expected.Title {
+		t.Errorf("actual.Title = %q; want %q", actual.Title, expected.Title)
+	}
+	if actual.Description != expected.Description {
+		t.Errorf("actual.Description = %q; want %q", actual.Description, expected.Description)
+	}
+	if len(actual.ImageURLs) != 1 || actual.ImageURLs[0] != expected.ImageURLs[0] {
+		t.Errorf("actual.ImageURLs = %q; want %q", actual.ImageURLs, expected.ImageURLs)
+	}
+}
+
+func genNostrKey() string {
+	k := nostr.GeneratePrivateKey()
+	if k == "" {
+		panic("nostr.GeneratePrivateKey returned empty string")
+	}
+	return k
+}
+
+func nostrPubKey(priv string) string {
+	pub, err := nostr.GetPublicKey(priv)
+	if err != nil {
+		panic(err.Error())
+	}
+	return pub
+}
+
+type FakeNostrRelay struct {
+	Event *nostr.Event
+
+	URL        string
+	HTTPServer *httptest.Server
+
+	Mu   sync.Mutex
+	Subs map[string]bool // id => true if still active; false for unsub'ed IDs
+}
+
+func (nr *FakeNostrRelay) Close() {
+	nr.HTTPServer.Close()
+}
+
+func (nr *FakeNostrRelay) OpenSubs() []string {
+	nr.Mu.Lock()
+	defer nr.Mu.Unlock()
+	var a []string
+	for k, open := range nr.Subs {
+		if open {
+			a = append(a, k)
+		}
+	}
+	return a
+}
+
+func nostrHandler(t *testing.T, nr *FakeNostrRelay) func(*websocket.Conn) {
+	return func(conn *websocket.Conn) {
+		for {
+			var req [3]any
+			if err := websocket.JSON.Receive(conn, &req); err != nil {
+				conn.Close()
+				return
+			}
+			switch req[0].(string) {
+			default:
+				t.Errorf("ws handler req[0]=%q; want REQ or CLOSE", req[0])
+				conn.Close()
+				return
+			case "CLOSE":
+				nr.Mu.Lock()
+				defer nr.Mu.Unlock()
+				nr.Subs[req[1].(string)] = false
+				return
+			case "REQ":
+				subid := req[1].(string)
+				nr.Mu.Lock()
+				nr.Subs[subid] = true
+				nr.Mu.Unlock()
+
+				filters := req[2].(map[string]any)
+				t.Logf("ws handler sub=%q, filters=%s", subid, filters)
+				if ids := filters["ids"].([]any); len(ids) != 1 || ids[0].(string) != nr.Event.ID {
+					t.Errorf("ws handler REQ filter ids=%q; want [%q]", ids, []string{nr.Event.ID})
+				}
+				if limit := filters["limit"].(float64); math.Abs(limit-1) > 0.00001 {
+					t.Errorf("ws handler REQ limit=%f; want 1", limit)
+				}
+				b, err := json.Marshal(nr.Event)
+				if err != nil {
+					t.Errorf("json.Marshal: %v", err)
+					conn.Close()
+					return
+				}
+				resp := fmt.Sprintf(`["EVENT", %q, %s]`, subid, b)
+				t.Logf("ws handler resp: %s", resp)
+				if err := websocket.Message.Send(conn, resp); err != nil {
+					t.Errorf("ws handler REQ write: %v", err)
+				}
+			}
+		}
+	}
+}
+
+func ServeSingleEvent(t *testing.T, event *nostr.Event) *FakeNostrRelay {
+	relay := &FakeNostrRelay{
+		Event: event,
+		Subs:  make(map[string]bool),
+	}
+	relay.HTTPServer = httptest.NewServer(&websocket.Server{
+		Handshake: func(conf *websocket.Config, r *http.Request) error {
+			t.Logf("new handshake from %s", r.RemoteAddr)
+			return nil
+		},
+		Handler: nostrHandler(t, relay),
+	})
+	tsurl, err := url.Parse(relay.HTTPServer.URL)
+	if err != nil {
+		panic(err)
+	}
+	relay.URL = fmt.Sprintf("ws://%s/", tsurl.Host)
+	return relay
+}

tools/release.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+
+VERSION=${VERSION:-$(git describe --tags)}
+export CGO_ENABLED=0
+exec go build -ldflags "-s -w -X main.noxyVersion=$VERSION" -buildmode=pie -trimpath ./cmd/noxy/