Merge remote-tracking branch 'public-gh/master' into paperclip-subissues

* public-gh/master:
  Drop lockfile from watcher change
  Tighten plugin dev file watching
  Fix plugin smoke example typecheck
  Fix plugin dev watcher and migration snapshot
  Clarify plugin authoring and external dev workflow
  Expand kitchen sink plugin demos
  fix: set AGENT_HOME env var for agent processes
  Add kitchen sink plugin example
  Simplify plugin runtime and cleanup lifecycle
  Add plugin framework and settings UI

# Conflicts:
#	packages/db/src/migrations/meta/0029_snapshot.json
#	packages/db/src/migrations/meta/_journal.json

server/src/app.ts

@@ -25,7 +25,23 @@ import { sidebarBadgeRoutes } from "./routes/sidebar-badges.js";
 import { llmRoutes } from "./routes/llms.js";
 import { assetRoutes } from "./routes/assets.js";
 import { accessRoutes } from "./routes/access.js";
+import { pluginRoutes } from "./routes/plugins.js";
+import { pluginUiStaticRoutes } from "./routes/plugin-ui-static.js";
 import { applyUiBranding } from "./ui-branding.js";
+import { logger } from "./middleware/logger.js";
+import { DEFAULT_LOCAL_PLUGIN_DIR, pluginLoader } from "./services/plugin-loader.js";
+import { createPluginWorkerManager } from "./services/plugin-worker-manager.js";
+import { createPluginJobScheduler } from "./services/plugin-job-scheduler.js";
+import { pluginJobStore } from "./services/plugin-job-store.js";
+import { createPluginToolDispatcher } from "./services/plugin-tool-dispatcher.js";
+import { pluginLifecycleManager } from "./services/plugin-lifecycle.js";
+import { createPluginJobCoordinator } from "./services/plugin-job-coordinator.js";
+import { buildHostServices, flushPluginLogBuffer } from "./services/plugin-host-services.js";
+import { createPluginEventBus } from "./services/plugin-event-bus.js";
+import { createPluginDevWatcher } from "./services/plugin-dev-watcher.js";
+import { createPluginHostServiceCleanup } from "./services/plugin-host-service-cleanup.js";
+import { pluginRegistryService } from "./services/plugin-registry.js";
+import { createHostClientHandlers } from "@paperclipai/plugin-sdk";
 import type { BetterAuthSessionResult } from "./auth/better-auth.js";
 
 type UiMode = "none" | "static" | "vite-dev";
@@ -49,13 +65,20 @@ export async function createApp(
     bindHost: string;
     authReady: boolean;
     companyDeletionEnabled: boolean;
+    instanceId?: string;
+    hostVersion?: string;
+    localPluginDir?: string;
     betterAuthHandler?: express.RequestHandler;
     resolveSession?: (req: ExpressRequest) => Promise<BetterAuthSessionResult | null>;
   },
 ) {
   const app = express();
 
-  app.use(express.json());
+  app.use(express.json({
+    verify: (req, _res, buf) => {
+      (req as unknown as { rawBody: Buffer }).rawBody = buf;
+    },
+  }));
   app.use(httpLogger);
   const privateHostnameGateEnabled =
     opts.deploymentMode === "authenticated" && opts.deploymentExposure === "private";
@@ -123,6 +146,68 @@ export async function createApp(
   api.use(activityRoutes(db));
   api.use(dashboardRoutes(db));
   api.use(sidebarBadgeRoutes(db));
+  const hostServicesDisposers = new Map<string, () => void>();
+  const workerManager = createPluginWorkerManager();
+  const pluginRegistry = pluginRegistryService(db);
+  const eventBus = createPluginEventBus();
+  const jobStore = pluginJobStore(db);
+  const lifecycle = pluginLifecycleManager(db, { workerManager });
+  const scheduler = createPluginJobScheduler({
+    db,
+    jobStore,
+    workerManager,
+  });
+  const toolDispatcher = createPluginToolDispatcher({
+    workerManager,
+    lifecycleManager: lifecycle,
+    db,
+  });
+  const jobCoordinator = createPluginJobCoordinator({
+    db,
+    lifecycle,
+    scheduler,
+    jobStore,
+  });
+  const hostServiceCleanup = createPluginHostServiceCleanup(lifecycle, hostServicesDisposers);
+  const loader = pluginLoader(
+    db,
+    { localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR },
+    {
+      workerManager,
+      eventBus,
+      jobScheduler: scheduler,
+      jobStore,
+      toolDispatcher,
+      lifecycleManager: lifecycle,
+      instanceInfo: {
+        instanceId: opts.instanceId ?? "default",
+        hostVersion: opts.hostVersion ?? "0.0.0",
+      },
+      buildHostHandlers: (pluginId, manifest) => {
+        const notifyWorker = (method: string, params: unknown) => {
+          const handle = workerManager.getWorker(pluginId);
+          if (handle) handle.notify(method, params);
+        };
+        const services = buildHostServices(db, pluginId, manifest.id, eventBus, notifyWorker);
+        hostServicesDisposers.set(pluginId, () => services.dispose());
+        return createHostClientHandlers({
+          pluginId,
+          capabilities: manifest.capabilities,
+          services,
+        });
+      },
+    },
+  );
+  api.use(
+    pluginRoutes(
+      db,
+      loader,
+      { scheduler, jobStore },
+      { workerManager },
+      { toolDispatcher },
+      { workerManager },
+    ),
+  );
   api.use(
     accessRoutes(db, {
       deploymentMode: opts.deploymentMode,
@@ -135,6 +220,9 @@ export async function createApp(
   app.use("/api", (_req, res) => {
     res.status(404).json({ error: "API route not found" });
   });
+  app.use(pluginUiStaticRoutes(db, {
+    localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
+  }));
 
   const __dirname = path.dirname(fileURLToPath(import.meta.url));
   if (opts.uiMode === "static") {
@@ -188,5 +276,35 @@ export async function createApp(
 
   app.use(errorHandler);
 
+  jobCoordinator.start();
+  scheduler.start();
+  void toolDispatcher.initialize().catch((err) => {
+    logger.error({ err }, "Failed to initialize plugin tool dispatcher");
+  });
+  const devWatcher = opts.uiMode === "vite-dev"
+    ? createPluginDevWatcher(
+      lifecycle,
+      async (pluginId) => (await pluginRegistry.getById(pluginId))?.packagePath ?? null,
+    )
+    : null;
+  void loader.loadAll().then((result) => {
+    if (!result) return;
+    for (const loaded of result.results) {
+      if (devWatcher && loaded.success && loaded.plugin.packagePath) {
+        devWatcher.watch(loaded.plugin.id, loaded.plugin.packagePath);
+      }
+    }
+  }).catch((err) => {
+    logger.error({ err }, "Failed to load ready plugins on startup");
+  });
+  process.once("exit", () => {
+    devWatcher?.close();
+    hostServiceCleanup.disposeAll();
+    hostServiceCleanup.teardown();
+  });
+  process.once("beforeExit", () => {
+    void flushPluginLogBuffer();
+  });
+
   return app;
 }