nandi/glean · 56b717c

modified .env.example +1 -1 Viewed

@@ -3,7 +3,7 @@ GLEAN_DB=glean.db
3	3	GLEAN_SESSION_KEY=change-me-to-a-random-string
4	4	GLEAN_JETSTREAM=wss://jetstream.glean.at
5	5	GLEAN_PLC_URL=https://didplc.glean.at
6		-GLEAN_SYNC_INTERVAL=30m
	6	+GLEAN_SYNC_INTERVAL=24h
7	7	GLEAN_CLUSTER_INTERVAL=60m
8	8	GLEAN_FETCH_INTERVAL=15m
9	9	GLEAN_COLLECTION_DIR_URL=https://lightrail.microcosm.blue/xrpc/com.atproto.sync.listReposByCollection?collection=at.glean.subscription

  GLEAN_SESSION_KEY=change-me-to-a-random-string
  GLEAN_JETSTREAM=wss://jetstream.glean.at
  GLEAN_PLC_URL=https://didplc.glean.at
--GLEAN_SYNC_INTERVAL=30m
++GLEAN_SYNC_INTERVAL=24h
  GLEAN_CLUSTER_INTERVAL=60m
  GLEAN_FETCH_INTERVAL=15m
  GLEAN_COLLECTION_DIR_URL=https://lightrail.microcosm.blue/xrpc/com.atproto.sync.listReposByCollection?collection=at.glean.subscription

modified internal/server/server.go +4 -0 Viewed

@@ -395,6 +395,10 @@ func (s Server) syncUserInBackground(userDID string, client atproto.Client) {
395	395	}()
396	396	}
397	397
	398	+// PeriodicSync runs a full PDS sync for all users on a fixed interval.
	399	+// Jetstream handles real-time create/update/delete events, but gaps can
	400	+// appear after Jetstream downtime or rare missed events. This catches up by
	401	+// reconciling each user's PDS records against the local index.
398	402	func (s *Server) PeriodicSync(ctx context.Context, interval time.Duration) {
399	403	ticker := time.NewTicker(interval)
400	404	defer ticker.Stop()

  	}()
  }
++// PeriodicSync runs a full PDS sync for all users on a fixed interval.
++// Jetstream handles real-time create/update/delete events, but gaps can
++// appear after Jetstream downtime or rare missed events. This catches up by
++// reconciling each user's PDS records against the local index.
  func (s *Server) PeriodicSync(ctx context.Context, interval time.Duration) {
  	ticker := time.NewTicker(interval)
  	defer ticker.Stop()

modified main.go +1 -1 Viewed

@@ -25,7 +25,7 @@ func main() {
25	25	addr := flag.String("addr", envOr("GLEAN_ADDR", ":8080"), "listen address")
26	26	dbPath := flag.String("db", envOr("GLEAN_DB", "glean.db"), "database path")
27	27	jetstreamURL := flag.String("jetstream", envOr("GLEAN_JETSTREAM", "wss://jetstream.glean.at"), "Jetstream URL")
28		- syncInterval := flag.Duration("sync-interval", envDuration("GLEAN_SYNC_INTERVAL", 30*time.Minute), "PDS sync interval")
	28	+ syncInterval := flag.Duration("sync-interval", envDuration("GLEAN_SYNC_INTERVAL", 24*time.Hour), "PDS sync interval")
29	29	clusterInterval := flag.Duration("cluster-interval", envDuration("GLEAN_CLUSTER_INTERVAL", 1*time.Hour), "cluster recomputation interval")
30	30	fetchInterval := flag.Duration("fetch-interval", envDuration("GLEAN_FETCH_INTERVAL", 15*time.Minute), "feed fetch tick interval")
31	31	collectionDirURL := flag.String("collection-dir", envOr("GLEAN_COLLECTION_DIR_URL", ""), "collection directory URL for startup backfill")

  	addr := flag.String("addr", envOr("GLEAN_ADDR", ":8080"), "listen address")
  	dbPath := flag.String("db", envOr("GLEAN_DB", "glean.db"), "database path")
  	jetstreamURL := flag.String("jetstream", envOr("GLEAN_JETSTREAM", "wss://jetstream.glean.at"), "Jetstream URL")
--	syncInterval := flag.Duration("sync-interval", envDuration("GLEAN_SYNC_INTERVAL", 30*time.Minute), "PDS sync interval")
++	syncInterval := flag.Duration("sync-interval", envDuration("GLEAN_SYNC_INTERVAL", 24*time.Hour), "PDS sync interval")
  	clusterInterval := flag.Duration("cluster-interval", envDuration("GLEAN_CLUSTER_INTERVAL", 1*time.Hour), "cluster recomputation interval")
  	fetchInterval := flag.Duration("fetch-interval", envDuration("GLEAN_FETCH_INTERVAL", 15*time.Minute), "feed fetch tick interval")
  	collectionDirURL := flag.String("collection-dir", envOr("GLEAN_COLLECTION_DIR_URL", ""), "collection directory URL for startup backfill")

modified readme.md +1 -1 Viewed

@@ -57,7 +57,7 @@ Then open `http://localhost:8080`.
57	57	\| `GLEAN_ADDR` \| `:8080` \| Listen address \|
58	58	\| `GLEAN_DB` \| `glean.db` \| SQLite base path (`_users`, `_articles`, `_recs` suffixes) \|
59	59	\| `GLEAN_JETSTREAM` \| `wss://jetstream.glean.at` \| Jetstream WebSocket URL \|
60		-\| `GLEAN_SYNC_INTERVAL` \| `30m` \| PDS sync interval (Go duration: `30m`, `2h30m`, etc.) \|
	60	+\| `GLEAN_SYNC_INTERVAL` \| `24h` \| PDS sync interval (Go duration: `24h`, `12h`, etc.) \|
61	61	\| `GLEAN_CLUSTER_INTERVAL` \| `1h` \| Cluster recomputation interval (Go duration) \|
62	62	\| `GLEAN_FETCH_INTERVAL` \| `15m` \| Feed fetch scheduler tick interval (Go duration) \|
63	63	\| `GLEAN_COLLECTION_DIR_URL` \| _(empty)_ \| Collection directory URL for startup backfill \|

  | `GLEAN_ADDR`                 | `:8080`                    | Listen address                                             |
  | `GLEAN_DB`                   | `glean.db`                 | SQLite base path (`_users`, `_articles`, `_recs` suffixes) |
  | `GLEAN_JETSTREAM`            | `wss://jetstream.glean.at` | Jetstream WebSocket URL                                    |
--| `GLEAN_SYNC_INTERVAL`        | `30m`                      | PDS sync interval (Go duration: `30m`, `2h30m`, etc.)      |
++| `GLEAN_SYNC_INTERVAL`        | `24h`                      | PDS sync interval (Go duration: `24h`, `12h`, etc.)        |
  | `GLEAN_CLUSTER_INTERVAL`     | `1h`                       | Cluster recomputation interval (Go duration)               |
  | `GLEAN_FETCH_INTERVAL`       | `15m`                      | Feed fetch scheduler tick interval (Go duration)           |
  | `GLEAN_COLLECTION_DIR_URL`   | _(empty)_                  | Collection directory URL for startup backfill              |

Increase PDS sync interval to 24 hoursUnverified