feat: integrate mailing list service (CM-1318)

.github/workflows/backend-lint.yaml

@@ -77,7 +77,7 @@ jobs:
       - name: Check for Python file changes
         id: changes
         run: |
-          if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q "^services/apps/git_integration/.*\.py$"; then
+          if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -qE "^services/apps/git_integration/.*(\.py|pyproject\.toml|uv\.lock)$"; then
             echo "python_changed=true" >> $GITHUB_OUTPUT
           else
             echo "python_changed=false" >> $GITHUB_OUTPUT
@@ -104,6 +104,53 @@ jobs:
           uv run ruff check src/ --output-format=github
           uv run ruff format --check src/
 
+  lint-python-mailing-list-integration:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./services/apps/mailing_list_integration
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check for Python file changes
+        id: changes
+        run: |
+          if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -qE "^services/apps/mailing_list_integration/.*(\.py|pyproject\.toml|uv\.lock)$"; then
+            echo "python_changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "python_changed=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Install uv
+        if: steps.changes.outputs.python_changed == 'true'
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "services/apps/mailing_list_integration/uv.lock"
+
+      - name: Set up Python
+        if: steps.changes.outputs.python_changed == 'true'
+        run: uv python install 3.13
+
+      - name: Install dependencies
+        if: steps.changes.outputs.python_changed == 'true'
+        run: uv sync --group dev --frozen
+
+      - name: Check Python linting and formatting
+        if: steps.changes.outputs.python_changed == 'true'
+        run: |
+          uv run ruff check src/ --output-format=github
+          uv run ruff format --check src/
+
+      - name: Run tests
+        if: steps.changes.outputs.python_changed == 'true'
+        run: uv run pytest src/test/ -v
+
       - name: Skip Python checks
         if: steps.changes.outputs.python_changed == 'false'
         run: echo "⏭️ No Python files changed, skipping linting checks"

backend/package.json

@@ -33,7 +33,11 @@
     "script:fix-duplicate-members": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/fix-duplicate-members.ts",
     "script:fix-members-activities-after-unaffilation": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/fix-members-activities-after-unaffilation.ts",
     "script:process-bot-members": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/process-bot-members.ts",
-    "script:backfill-email-domain-member-organization-dates": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/backfill-email-domain-member-organization-dates.ts"
+    "script:backfill-email-domain-member-organization-dates": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/backfill-email-domain-member-organization-dates.ts",
+    "script:onboard-default-tenant": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/onboard-default-tenant.ts",
+    "script:onboard-default-tenant:local": "set -a && . ./.env.dist.local && . ./.env.override.local && set +a && pnpm run script:onboard-default-tenant",
+    "script:create-mailing-list-integration": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/create-mailing-list-integration.ts",
+    "script:create-mailing-list-integration:local": "set -a && . ./.env.dist.local && . ./.env.override.local && set +a && SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/create-mailing-list-integration.ts"
   },
   "lint-staged": {
     "**/*.ts": [

backend/src/api/integration/helpers/mailingListAuthenticate.ts

@@ -0,0 +1,93 @@
+import { z } from 'zod'
+
+import Permissions from '../../../security/permissions'
+import IntegrationService from '../../../services/integrationService'
+import PermissionChecker from '../../../services/user/permissionChecker'
+import { validateOrThrow } from '../../../utils/validation'
+
+const MAX_LIST_NAME_LENGTH = 255
+
+// `name` is a display label only — the worker keys the mirror path by the
+// DB-generated list id, not `name` (see mirror_service.py's
+// list_mirror_dir/_validate_list_id), so it's free to contain path-like
+// characters (lore lists are nested, e.g. "some-project/git"). Only reject
+// the null byte, which breaks C-string-based tooling downstream regardless
+// of context.
+const isSafeListName = (name: string): boolean =>
+  name.length > 0 && name.length <= MAX_LIST_NAME_LENGTH && !name.includes('\0')
+
+// public-inbox-clone in the worker fetches this URL as-is; restrict to
+// https so a caller can't point the worker at file://, javascript:, or a
+// bare non-URL string. Requiring https already blocks the classic SSRF
+// target (cloud-metadata IMDS is http-only, per securityTxt.ts precedent);
+// also reject obvious loopback/localhost literals.
+const isBlockedHost = (h: string): boolean =>
+  h === 'localhost' || h === '::1' || h === '0.0.0.0' || h.startsWith('127.')
+
+const isSafeSourceUrl = (sourceUrl: string): boolean => {
+  try {
+    const url = new URL(sourceUrl)
+    return (
+      url.protocol === 'https:' &&
+      !isBlockedHost(url.hostname.toLowerCase()) &&
+      // Credentials/query/fragment have no meaning for a public-inbox archive
+      // URL; rejecting them keeps canonicalizeSourceUrl a lossless
+      // normalization instead of one that silently drops caller-supplied data.
+      url.username === '' &&
+      url.password === '' &&
+      url.search === '' &&
+      url.hash === ''
+    )
+  } catch {
+    return false
+  }
+}
+
+// "https://host/list" and "https://HOST/list/" and "https://host:443/list"
+// must all resolve to the same DB row — scheme case, host case, the default
+// https port, and a trailing slash are not meaningful differences for the
+// same archive, but a plain trailing-slash strip left them as distinct
+// strings, bypassing the cross-project ownership check and causing
+// duplicate ingestion. The worker's ensure_mirror() already normalizes to
+// this same form before cloning (mirror_service.py), so storage must match.
+// Kept as a plain function (not a zod .transform()/.preprocess()) since
+// either makes the field optional in z.infer with the installed zod v4 —
+// reproduced in isolation, unrelated to this schema's nesting.
+export const canonicalizeSourceUrl = (sourceUrl: string): string => {
+  const url = new URL(sourceUrl)
+  const port = url.port && url.port !== '443' ? `:${url.port}` : ''
+  const path = url.pathname.replace(/\/+$/, '') || '/'
+  return `https://${url.hostname.toLowerCase()}${port}${path}`
+}
+
+export const bodySchema = z.object({
+  lists: z
+    .array(
+      z.object({
+        name: z.string().trim().min(1).refine(isSafeListName, {
+          message: 'Invalid mailing list name',
+        }),
+        sourceUrl: z.string().trim().min(1).refine(isSafeSourceUrl, {
+          message: 'sourceUrl must be a valid https:// URL',
+        }),
+      }),
+    )
+    .min(1, 'lists must contain at least one mailing list')
+    .refine(
+      (lists) =>
+        new Set(lists.map((l) => canonicalizeSourceUrl(l.sourceUrl))).size === lists.length,
+      { message: 'lists contains duplicate sourceUrl entries' },
+    ),
+})
+
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.tenantEdit)
+  const integrationData = validateOrThrow(bodySchema, req.body)
+  integrationData.lists = integrationData.lists.map((l) => ({
+    ...l,
+    sourceUrl: canonicalizeSourceUrl(l.sourceUrl),
+  }))
+
+  const payload = await new IntegrationService(req).mailingListConnectOrUpdate(integrationData)
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/integration/index.ts

@@ -77,6 +77,8 @@ export default (app) => {
 
   // Git
   app.put(`/git-connect`, safeWrap(require('./helpers/gitAuthenticate').default))
+
+  app.put(`/mailing-list-connect`, safeWrap(require('./helpers/mailingListAuthenticate').default))
   app.put(`/confluence-connect`, safeWrap(require('./helpers/confluenceAuthenticate').default))
   app.put(`/gerrit-connect`, safeWrap(require('./helpers/gerritAuthenticate').default))
   app.get('/devto-validate', safeWrap(require('./helpers/devtoValidators').default))

backend/src/bin/scripts/create-mailing-list-integration.ts

@@ -0,0 +1,124 @@
+/* eslint-disable no-console */
+
+/* eslint-disable import/no-extraneous-dependencies */
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import * as fs from 'fs'
+
+import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common'
+
+import {
+  bodySchema,
+  canonicalizeSourceUrl,
+} from '@/api/integration/helpers/mailingListAuthenticate'
+import SegmentRepository from '@/database/repositories/segmentRepository'
+import SequelizeRepository from '@/database/repositories/sequelizeRepository'
+import IntegrationService from '@/services/integrationService'
+import { validateOrThrow } from '@/utils/validation'
+
+const options = [
+  {
+    name: 'help',
+    alias: 'h',
+    type: Boolean,
+    description: 'Print this usage guide.',
+  },
+  {
+    name: 'file',
+    alias: 'f',
+    type: String,
+    description:
+      'Path to a JSON file with the lists array, each entry shaped as name + sourceUrl ' +
+      '(e.g. name "linux-serial", sourceUrl https://lore.kernel.org/linux-serial).',
+  },
+  {
+    name: 'segment',
+    alias: 's',
+    type: String,
+    description: "Segment id. Optional — defaults to the tenant's first subproject.",
+  },
+]
+
+const sections = [
+  {
+    header: 'Create mailing list integration',
+    content:
+      'Calls IntegrationService.mailingListConnectOrUpdate() directly to create/update the ' +
+      'mailinglist integration and onboard its lists for processing, until the frontend connect ' +
+      'UI ships (CM-1318).',
+  },
+  {
+    header: 'Options',
+    optionList: options,
+  },
+]
+
+const usage = commandLineUsage(sections)
+// pnpm forwards a literal `--` separator when args are passed via `pnpm run ... -- <args>`;
+// strip it so command-line-args doesn't choke on it.
+const argv = process.argv.slice(2).filter((arg) => arg !== '--')
+const parameters = commandLineArgs(options, { argv })
+
+if (parameters.help || !parameters.file) {
+  console.log(usage)
+  process.exit(parameters.help ? 0 : 1)
+} else {
+  setImmediate(async () => {
+    let fileContents: string
+    try {
+      fileContents = fs.readFileSync(parameters.file, 'utf-8')
+    } catch (err) {
+      console.error(`Could not read file at ${parameters.file}: ${(err as Error).message}`)
+      process.exit(1)
+    }
+
+    let parsed: unknown
+    try {
+      parsed = JSON.parse(fileContents)
+    } catch (err) {
+      console.error(`File at ${parameters.file} is not valid JSON: ${(err as Error).message}`)
+      process.exit(1)
+    }
+
+    // Same validation as the connect API endpoint (mailingListAuthenticate.ts),
+    // so a malformed file (non-array, missing fields, duplicate/non-https
+    // sourceUrl) fails fast here instead of reaching IntegrationService with
+    // whatever shape the file happened to contain.
+    const { lists: parsedLists } = validateOrThrow(bodySchema, { lists: parsed })
+    const lists = parsedLists.map((l) => ({ ...l, sourceUrl: canonicalizeSourceUrl(l.sourceUrl) }))
+
+    const repoOptions = await SequelizeRepository.getDefaultIRepositoryOptions()
+    repoOptions.currentTenant = { id: DEFAULT_TENANT_ID }
+    ;(repoOptions as unknown as { requestId: string }).requestId = generateUUIDv1()
+
+    const adminUser = await repoOptions.database.user.findOne({ where: {} })
+    if (!adminUser) {
+      console.error('No user found — run script:onboard-default-tenant first.')
+      process.exit(1)
+    }
+    repoOptions.currentUser = adminUser
+
+    const segmentRepository = new SegmentRepository(repoOptions)
+    repoOptions.currentSegments = parameters.segment
+      ? await segmentRepository.findInIds([parameters.segment])
+      : (await segmentRepository.querySubprojects({ limit: 1, offset: 0 })).rows
+
+    if (repoOptions.currentSegments.length === 0) {
+      console.error('No segment found/resolved — pass --segment explicitly.')
+      process.exit(1)
+    }
+
+    console.log(
+      `Segment: ${repoOptions.currentSegments[0].id} (${repoOptions.currentSegments[0].name})`,
+    )
+    console.log('Lists:', JSON.stringify(lists, null, 2))
+
+    const integration = await new IntegrationService(repoOptions).mailingListConnectOrUpdate(
+      { lists },
+      repoOptions,
+    )
+
+    console.log('Integration:', JSON.stringify(integration, null, 2))
+    process.exit(0)
+  })
+}

backend/src/bin/scripts/onboard-default-tenant.ts

@@ -0,0 +1,74 @@
+/* eslint-disable no-console */
+
+/* eslint-disable import/no-extraneous-dependencies */
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+
+import SequelizeRepository from '@/database/repositories/sequelizeRepository'
+import AuthService from '@/services/auth/authService'
+
+const options = [
+  {
+    name: 'help',
+    alias: 'h',
+    type: Boolean,
+    description: 'Print this usage guide.',
+  },
+  {
+    name: 'email',
+    alias: 'e',
+    type: String,
+    defaultValue: 'local-dev@example.com',
+    description: 'Email for the dev user driving the onboarding (default: local-dev@example.com)',
+  },
+]
+
+const sections = [
+  {
+    header: 'Onboard default tenant',
+    content:
+      'Runs the same onboarding logic as a real Auth0 signup (AuthService.signinFromSSO -> ' +
+      'handleOnboard -> TenantService.createOrJoinDefault), without needing a real Auth0 token. ' +
+      'Creates the default tenant, default segment, settings, and an admin user/tenantUser row. ' +
+      'Safe to re-run — createOrJoinDefault joins the existing tenant instead of duplicating it.',
+  },
+  {
+    header: 'Options',
+    optionList: options,
+  },
+]
+
+const usage = commandLineUsage(sections)
+const parameters = commandLineArgs(options)
+
+if (parameters.help) {
+  console.log(usage)
+} else {
+  setImmediate(async () => {
+    const repoOptions = await SequelizeRepository.getDefaultIRepositoryOptions()
+
+    await AuthService.signinFromSSO(
+      'auth0',
+      `dev|${parameters.email}`,
+      parameters.email,
+      true,
+      'Local',
+      'Dev',
+      'Local Dev',
+      null,
+      null,
+      null,
+      repoOptions,
+    )
+
+    const tenant = await repoOptions.database.tenant.findOne({ where: {} })
+    const segment = await repoOptions.database.segment.findOne({
+      where: { tenantId: tenant.id },
+    })
+
+    console.log(`Tenant: ${tenant.id} (${tenant.name})`)
+    console.log(`Segment: ${segment.id} (${segment.name})`)
+
+    process.exit(0)
+  })
+}