local.dev Tunnel Service - Design Analysis & Revised Plan

┌─────────────────────────────────────┐
│   Fly Edge → Gateway App            │
│            ↓                         │
│   Per-Customer Fly Machines         │
│   (CoreDNS + Caddy + Node Proxy)    │
└─────────────────────────────────────┘

// From PRD workspace-manager.js
async createCustomerWorkspace(customer, tier) {
  const machine = await this.client.create({
    // Creates NEW machine per customer
    config: {
      image: 'localdev/workspace:latest',
      // Each has CoreDNS, Caddy, Node - duplicated infrastructure
    }
  });
}

// From PRD cli.js - synchronous request handling
async forwardToLocal(request) {
  return new Promise((resolve) => {
    const req = http.request({ /* ... */ }, (res) => {
      // Blocks until complete - no concurrency
    });
  });
}

┌──────────────────────────────────────────────────────────┐
│                   STATELESS GATEWAY TIER                  │
│  ┌────────────────────────────────────────────────────┐  │
│  │  Fly.io Gateway Instances (Horizontal Scaling)     │  │
│  │  - WS Connection Handling                          │  │
│  │  - TLS Termination (Caddy/Let's Encrypt)          │  │
│  │  - HTTP Proxy Logic                                │  │
│  │  - Session Routing via Redis                       │  │
│  └────────────────────────────────────────────────────┘  │
└──────────────────────────────────────────────────────────┘
                          ↓
              ┌──────────────────────┐
              │   ROUTING REGISTRY   │
              │                      │
              │  Redis (Cache)       │
              │  subdomain → ws_id   │
              │                      │
              │  PostgreSQL (Source) │
              │  tunnels table       │
              └──────────────────────┘

1. Public HTTP(S) Request
   ↓
2. Gateway receives on subdomain (e.g., abc123.local.dev)
   ↓
3. Lookup in Redis: subdomain → ws_connection_id
   ↓
4. If not in Redis, query PostgreSQL, cache result
   ↓
5. Send request to WS connection (identified by ws_connection_id)
   ↓
6. CLI client receives request, forwards to localhost:PORT
   ↓
7. CLI client sends response back via WS
   ↓
8. Gateway streams response to original HTTP requester

// Client → Gateway (tunnel registration)
{
  type: 'register',
  apiKey: string,
  localPort: number,
  requestedSubdomain?: string  // Optional custom subdomain
}

// Gateway → Client (registration success)
{
  type: 'registered',
  tunnelId: string,
  subdomain: string,
  publicUrl: string
}

// Gateway → Client (HTTP request)
{
  type: 'http_request',
  requestId: string,
  method: string,
  path: string,
  headers: Record<string, string>,
  body?: string
}

// Client → Gateway (HTTP response)
{
  type: 'http_response',
  requestId: string,
  statusCode: number,
  headers: Record<string, string>,
  body: string
}

// Heartbeat (bidirectional)
{
  type: 'ping' | 'pong',
  timestamp: number
}

// Client-side concurrent request handling
class TunnelClient {
  constructor() {
    this.pendingRequests = new Map(); // requestId → Promise
    this.requestQueue = [];
    this.maxConcurrent = 50; // Handle 50 concurrent requests
  }

  async handleIncomingRequest(message) {
    // Non-blocking: add to queue, process concurrently
    this.requestQueue.push(message);
    this.processQueue();
  }

  async processQueue() {
    while (this.requestQueue.length > 0 &&
           this.pendingRequests.size < this.maxConcurrent) {
      const request = this.requestQueue.shift();

      // Fire and forget - doesn't block
      this.forwardToLocal(request).catch(err => {
        this.handleError(request.id, err);
      });
    }
  }
}

# DNS records at your registrar (e.g., Cloudflare, Route53)
A     local.dev           →  [Fly.io IPv4 from: fly ips list]
AAAA  local.dev           →  [Fly.io IPv6 from: fly ips list]
CNAME *.local.dev         →  localdev-gateway.fly.dev

# Fly.io automatically routes ALL subdomains to gateway:
# - abc123.local.dev → Gateway
# - pr-456.local.dev → Gateway
# - anything.local.dev → Gateway

// Gateway extracts subdomain and routes to correct tunnel
const express = require('express');
const app = express();

app.use(async (req, res, next) => {
  // Extract subdomain from hostname
  const hostname = req.hostname; // e.g., "abc123.local.dev"
  const subdomain = hostname.replace('.local.dev', ''); // "abc123"

  // Look up which WebSocket connection owns this subdomain
  let wsConnectionId = await redis.get(`subdomain:${subdomain}`);

  if (!wsConnectionId) {
    // Not in cache, check database
    const tunnel = await db.query(
      'SELECT ws_connection_id FROM tunnels WHERE subdomain = $1 AND status = $2',
      [subdomain, 'active']
    );

    if (!tunnel) {
      return res.status(404).send('Tunnel not found');
    }

    wsConnectionId = tunnel.ws_connection_id;

    // Cache for 60 seconds
    await redis.setex(`subdomain:${subdomain}`, 60, wsConnectionId);
  }

  // Attach to request for proxy middleware
  req.tunnelConnectionId = wsConnectionId;
  next();
});

// Proxy middleware handles actual forwarding
app.use(proxyMiddleware);

1. User adds custom domain in dashboard: "their-app.com"
2. System generates DNS verification record
3. User adds CNAME record at their registrar
4. System validates DNS record exists
5. System provisions SSL certificate via Caddy
6. Requests to their-app.com → Gateway → Tunnel

// POST /api/domains - User adds custom domain
app.post('/api/domains', authenticate, async (req, res) => {
  const { domain } = req.body;
  const userId = req.user.id;

  // Generate verification token
  const verificationToken = crypto.randomBytes(16).toString('hex');
  const verificationSubdomain = `_localdev-verify-${verificationToken}`;

  // Store in database
  await db.query(
    `INSERT INTO custom_domains (user_id, domain, validation_token, dns_verified)
     VALUES ($1, $2, $3, false)`,
    [userId, domain, verificationToken]
  );

  res.json({
    domain,
    status: 'pending_verification',
    dns_records: [
      {
        type: 'TXT',
        name: verificationSubdomain,
        value: `localdev-verification=${verificationToken}`,
        ttl: 300
      },
      {
        type: 'CNAME',
        name: domain,
        value: 'localdev-gateway.fly.dev',
        ttl: 3600
      }
    ],
    instructions: `Add these DNS records at your domain registrar, then click 'Verify'`
  });
});

// POST /api/domains/:domain/verify - Verify DNS
app.post('/api/domains/:domain/verify', authenticate, async (req, res) => {
  const { domain } = req.params;
  const userId = req.user.id;

  // Get verification token
  const record = await db.query(
    'SELECT validation_token FROM custom_domains WHERE user_id = $1 AND domain = $2',
    [userId, domain]
  );

  if (!record) {
    return res.status(404).json({ error: 'Domain not found' });
  }

  // Check DNS records
  const dns = require('dns').promises;
  const verificationSubdomain = `_localdev-verify-${record.validation_token}`;

  try {
    const txtRecords = await dns.resolveTxt(verificationSubdomain);
    const verified = txtRecords.some(records =>
      records.includes(`localdev-verification=${record.validation_token}`)
    );

    if (verified) {
      // Mark as verified
      await db.query(
        'UPDATE custom_domains SET dns_verified = true, verified_at = NOW() WHERE domain = $1',
        [domain]
      );

      // Trigger SSL certificate provisioning
      await provisionSSL(domain);

      return res.json({
        status: 'verified',
        message: 'Domain verified successfully. SSL certificate provisioning in progress.'
      });
    }
  } catch (err) {
    return res.status(400).json({
      error: 'DNS verification failed',
      details: 'TXT record not found or incorrect'
    });
  }
});

// Gateway routing for custom domains
app.use(async (req, res, next) => {
  const hostname = req.hostname;

  // Check if it's a subdomain OR custom domain
  let wsConnectionId;

  if (hostname.endsWith('.local.dev')) {
    // Subdomain routing (existing logic)
    const subdomain = hostname.replace('.local.dev', '');
    wsConnectionId = await redis.get(`subdomain:${subdomain}`);
  } else {
    // Custom domain routing
    wsConnectionId = await redis.get(`domain:${hostname}`);

    if (!wsConnectionId) {
      // Check database
      const domain = await db.query(
        `SELECT t.ws_connection_id
         FROM custom_domains cd
         JOIN tunnels t ON t.user_id = cd.user_id
         WHERE cd.domain = $1 AND cd.dns_verified = true AND t.status = 'active'
         LIMIT 1`,
        [hostname]
      );

      if (domain) {
        wsConnectionId = domain.ws_connection_id;
        await redis.setex(`domain:${hostname}`, 60, wsConnectionId);
      }
    }
  }

  if (!wsConnectionId) {
    return res.status(404).send('Tunnel not found');
  }

  req.tunnelConnectionId = wsConnectionId;
  next();
});

{
  # Automatic HTTPS for all domains
  email ssl@local.dev

  # On-demand TLS: issue certs as requests come in
  on_demand_tls {
    ask http://localhost:3000/caddy/cert-allowed
    interval 2m
    burst 5
  }
}

# Wildcard for all *.local.dev subdomains
*.local.dev {
  reverse_proxy localhost:3000

  tls {
    on_demand
  }
}

# Custom domains (validated before cert issuance)
:443 {
  reverse_proxy localhost:3000

  tls {
    on_demand
  }
}

// Caddy asks: "Should I issue a cert for this domain?"
app.get('/caddy/cert-allowed', async (req, res) => {
  const domain = req.query.domain;

  // Allow *.local.dev (our subdomains)
  if (domain.endsWith('.local.dev')) {
    return res.status(200).send('OK');
  }

  // Check if custom domain is verified
  const verified = await db.query(
    'SELECT 1 FROM custom_domains WHERE domain = $1 AND dns_verified = true',
    [domain]
  );

  if (verified) {
    return res.status(200).send('OK');
  }

  // Reject unknown domains
  res.status(403).send('Domain not allowed');
});

const WebSocket = require('ws');
const { v4: uuidv4 } = require('uuid');

class TunnelProxyMiddleware {
  constructor() {
    this.activeConnections = new Map(); // wsConnectionId → WebSocket
    this.pendingRequests = new Map(); // requestId → { resolve, reject, timeout }
  }

  // Register a WebSocket connection when CLI connects
  registerConnection(wsConnectionId, ws) {
    this.activeConnections.set(wsConnectionId, ws);

    // Handle responses from CLI client
    ws.on('message', (data) => {
      const message = JSON.parse(data);

      if (message.type === 'http_response') {
        this.handleResponse(message);
      }
    });

    ws.on('close', () => {
      this.activeConnections.delete(wsConnectionId);
    });
  }

  // Middleware function
  async handle(req, res, next) {
    const wsConnectionId = req.tunnelConnectionId;

    if (!wsConnectionId) {
      return res.status(500).send('No tunnel connection ID');
    }

    const ws = this.activeConnections.get(wsConnectionId);

    if (!ws || ws.readyState !== WebSocket.OPEN) {
      return res.status(503).send('Tunnel disconnected');
    }

    try {
      // Collect request body
      const chunks = [];
      req.on('data', chunk => chunks.push(chunk));
      req.on('end', async () => {
        const body = Buffer.concat(chunks);

        // Send HTTP request over WebSocket tunnel
        const response = await this.sendRequest(ws, {
          method: req.method,
          path: req.url,
          headers: req.headers,
          body: body.toString('base64')
        });

        // Stream response back to client
        res.status(response.statusCode);
        Object.entries(response.headers || {}).forEach(([key, value]) => {
          res.setHeader(key, value);
        });
        res.send(Buffer.from(response.body, 'base64'));
      });
    } catch (err) {
      if (err.message === 'Tunnel timeout') {
        return res.status(504).send('Tunnel timeout');
      }
      return res.status(500).send('Tunnel error');
    }
  }

  sendRequest(ws, requestData) {
    const requestId = uuidv4();

    return new Promise((resolve, reject) => {
      // Set 30-second timeout
      const timeout = setTimeout(() => {
        this.pendingRequests.delete(requestId);
        reject(new Error('Tunnel timeout'));
      }, 30000);

      // Store pending request
      this.pendingRequests.set(requestId, { resolve, reject, timeout });

      // Send request to CLI client via WebSocket
      ws.send(JSON.stringify({
        type: 'http_request',
        requestId,
        ...requestData
      }));
    });
  }

  handleResponse(message) {
    const { requestId, statusCode, headers, body } = message;
    const pending = this.pendingRequests.get(requestId);

    if (pending) {
      clearTimeout(pending.timeout);
      this.pendingRequests.delete(requestId);
      pending.resolve({ statusCode, headers, body });
    }
  }
}

// Usage in Express app
const proxyMiddleware = new TunnelProxyMiddleware();
app.use((req, res) => proxyMiddleware.handle(req, res));

const server = require('http').createServer(app);
const wss = new WebSocket.Server({ noServer: true });

server.on('upgrade', async (req, socket, head) => {
  const hostname = req.headers.host;
  const subdomain = hostname.replace('.local.dev', '');

  // Find tunnel connection
  const wsConnectionId = await redis.get(`subdomain:${subdomain}`);

  if (!wsConnectionId) {
    socket.destroy();
    return;
  }

  const tunnelWs = proxyMiddleware.activeConnections.get(wsConnectionId);

  if (!tunnelWs || tunnelWs.readyState !== WebSocket.OPEN) {
    socket.destroy();
    return;
  }

  // Notify CLI client about WebSocket upgrade
  const upgradeId = uuidv4();
  tunnelWs.send(JSON.stringify({
    type: 'websocket_upgrade',
    upgradeId,
    path: req.url,
    headers: req.headers
  }));

  // Accept the WebSocket upgrade on gateway side
  wss.handleUpgrade(req, socket, head, (clientWs) => {
    // Create bidirectional tunnel
    // User's WS ←→ Gateway ←→ Tunnel WS ←→ CLI ←→ localhost WS

    clientWs.on('message', (data) => {
      // Forward to tunnel
      tunnelWs.send(JSON.stringify({
        type: 'websocket_data',
        upgradeId,
        data: data.toString('base64')
      }));
    });

    clientWs.on('close', () => {
      tunnelWs.send(JSON.stringify({
        type: 'websocket_close',
        upgradeId
      }));
    });

    // Listen for data from tunnel
    const handleTunnelMessage = (msg) => {
      const message = JSON.parse(msg);
      if (message.upgradeId === upgradeId) {
        if (message.type === 'websocket_data') {
          clientWs.send(Buffer.from(message.data, 'base64'));
        } else if (message.type === 'websocket_close') {
          clientWs.close();
        }
      }
    };

    tunnelWs.on('message', handleTunnelMessage);
  });
});

┌─────────────┐
│   Browser   │ GET https://abc123.local.dev/api/users
└──────┬──────┘
       │ 1. DNS Lookup
       ↓
┌─────────────────┐
│  DNS Registrar  │ *.local.dev → CNAME → localdev-gateway.fly.dev
└────────┬────────┘
         │ 2. Resolves to Fly.io IP
         ↓
┌─────────────────────────────────────┐
│      Fly.io Anycast (Global)        │ Routes to nearest region
└─────────────┬───────────────────────┘
              │ 3. TLS termination (Caddy)
              ↓
┌─────────────────────────────────────┐
│         Gateway Instance            │
│  a) Extract: subdomain = "abc123"   │
│  b) Redis: subdomain:abc123 → ws_7  │
│  c) WebSocket send: http_request    │
└─────────────┬───────────────────────┘
              │ 4. WebSocket message
              ↓
┌─────────────────────────────────────┐
│         CLI Client (Developer)      │
│  a) Receive http_request            │
│  b) HTTP to localhost:3000          │
│  c) Get response                    │
│  d) WebSocket send: http_response   │
└─────────────┬───────────────────────┘
              │ 5. localhost:3000
              ↓
┌─────────────────────────────────────┐
│    Local Dev Server (Next.js/etc)   │ Returns JSON response
└─────────────────────────────────────┘

┌─────────────┐
│   Browser   │ GET https://their-app.com/
└──────┬──────┘
       │ 1. DNS Lookup
       ↓
┌─────────────────┐
│ User's Registrar│ CNAME → localdev-gateway.fly.dev
└────────┬────────┘
         │ 2. Resolves to Fly.io IP
         ↓
┌─────────────────────────────────────┐
│      Fly.io Gateway                 │
│  a) Caddy: Check SSL cert           │
│     - If exists: use it              │
│     - If not: ask /caddy/cert-allowed│
│     - Issue Let's Encrypt cert       │
│  b) Extract: hostname = "their-app.com"│
│  c) Redis: domain:their-app.com → ws│
│  d) Proxy via WebSocket (same flow) │
└─────────────────────────────────────┘

# .github/workflows/preview.yml
- name: Deploy PR Preview
  uses: localdev/preview-action@v1
  with:
    port: 3000
    subdomain: pr-${{ github.event.number }}
# Automatically comments PR with URL

-- =======================
-- CORE TABLES
-- =======================

-- Users (authentication)
CREATE TABLE users (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  email VARCHAR(255) UNIQUE NOT NULL,
  password_hash VARCHAR(255) NOT NULL,
  email_verified BOOLEAN DEFAULT FALSE,
  created_at TIMESTAMP DEFAULT NOW(),
  updated_at TIMESTAMP DEFAULT NOW()
);

-- API Keys
CREATE TABLE api_keys (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  user_id UUID REFERENCES users(id) ON DELETE CASCADE,
  name VARCHAR(255) NOT NULL,
  key_hash VARCHAR(255) UNIQUE NOT NULL,
  scopes TEXT[] DEFAULT ARRAY['tunnel:create', 'tunnel:read'],
  last_used_at TIMESTAMP,
  created_at TIMESTAMP DEFAULT NOW(),
  revoked_at TIMESTAMP
);

-- Subscriptions
CREATE TABLE subscriptions (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  user_id UUID REFERENCES users(id) ON DELETE CASCADE,
  tier VARCHAR(50) NOT NULL DEFAULT 'free',
  stripe_customer_id VARCHAR(255),
  stripe_subscription_id VARCHAR(255),
  status VARCHAR(50) DEFAULT 'active',
  current_period_start TIMESTAMP,
  current_period_end TIMESTAMP,
  created_at TIMESTAMP DEFAULT NOW(),
  updated_at TIMESTAMP DEFAULT NOW()
);

-- =======================
-- TUNNEL MANAGEMENT
-- =======================

-- Active Tunnels (volatile, cleared on restart)
CREATE TABLE tunnels (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  user_id UUID REFERENCES users(id) ON DELETE CASCADE,
  subdomain VARCHAR(255) UNIQUE NOT NULL,
  ws_connection_id VARCHAR(255) UNIQUE NOT NULL,
  local_port INTEGER NOT NULL,
  gateway_instance VARCHAR(255), -- Which gateway instance owns this
  status VARCHAR(50) DEFAULT 'active',
  created_at TIMESTAMP DEFAULT NOW(),
  last_heartbeat TIMESTAMP DEFAULT NOW(),
  closed_at TIMESTAMP
);

-- Tunnel Sessions (historical record)
CREATE TABLE tunnel_sessions (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  user_id UUID REFERENCES users(id),
  tunnel_id UUID,
  subdomain VARCHAR(255),
  duration_seconds INTEGER,
  bytes_transferred BIGINT DEFAULT 0,
  requests_count INTEGER DEFAULT 0,
  started_at TIMESTAMP DEFAULT NOW(),
  ended_at TIMESTAMP
);

-- Custom Domains
CREATE TABLE custom_domains (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  user_id UUID REFERENCES users(id) ON DELETE CASCADE,
  domain VARCHAR(255) UNIQUE NOT NULL,
  dns_verified BOOLEAN DEFAULT FALSE,
  ssl_enabled BOOLEAN DEFAULT FALSE,
  ssl_cert_id VARCHAR(255),
  validation_token VARCHAR(255),
  created_at TIMESTAMP DEFAULT NOW(),
  verified_at TIMESTAMP
);

-- =======================
-- USAGE & BILLING
-- =======================

-- Usage Metrics (aggregated daily)
CREATE TABLE usage_metrics (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  user_id UUID REFERENCES users(id),
  date DATE NOT NULL,
  metric_type VARCHAR(50) NOT NULL, -- 'tunnels_created', 'bandwidth', 'requests'
  value BIGINT NOT NULL,
  created_at TIMESTAMP DEFAULT NOW(),
  UNIQUE(user_id, date, metric_type)
);

-- =======================
-- TEAMS (for Team tier)
-- =======================

-- Teams
CREATE TABLE teams (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  name VARCHAR(255) NOT NULL,
  owner_id UUID REFERENCES users(id),
  subscription_id UUID REFERENCES subscriptions(id),
  created_at TIMESTAMP DEFAULT NOW()
);

-- Team Members
CREATE TABLE team_members (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  team_id UUID REFERENCES teams(id) ON DELETE CASCADE,
  user_id UUID REFERENCES users(id) ON DELETE CASCADE,
  role VARCHAR(50) DEFAULT 'member', -- 'owner', 'admin', 'member'
  joined_at TIMESTAMP DEFAULT NOW(),
  UNIQUE(team_id, user_id)
);

-- =======================
-- INDEXES
-- =======================

CREATE INDEX idx_tunnels_user ON tunnels(user_id);
CREATE INDEX idx_tunnels_subdomain ON tunnels(subdomain);
CREATE INDEX idx_tunnels_ws_conn ON tunnels(ws_connection_id);
CREATE INDEX idx_tunnels_status ON tunnels(status) WHERE status = 'active';

CREATE INDEX idx_sessions_user_date ON tunnel_sessions(user_id, started_at);
CREATE INDEX idx_usage_user_date ON usage_metrics(user_id, date);

CREATE INDEX idx_api_keys_user ON api_keys(user_id) WHERE revoked_at IS NULL;
CREATE INDEX idx_subscriptions_user ON subscriptions(user_id);