Examples
Onboarding redesign — full OES document
A complete v0.1 OES document for a realistic A/B test on new-account onboarding. It exercises every major section of the spec — design, variants, metrics, analysis, results, scorecard, decision, quality checks, artifacts, and provenance — so you can use it as a reference when producing your own documents.
{
"schemaVersion": "0.1.0",
"objectType": "experiment",
"exportedAt": "2026-04-18T14:32:00Z",
"sourceSystem": "GrowthBook",
"sourceSystemVersion": "3.4.1",
"canonicalUrl": "https://app.growthbook.io/experiment/exp_8d4a",
"externalIds": {
"jira": "GROW-1842",
"github_pr": "growthbook/web#9214"
},
"experiment": {
"id": "exp_8d4a",
"slug": "onboarding-redesign-2026q2",
"title": "Onboarding redesign — interactive checklist",
"summary": "Replace the static onboarding page with an interactive 4-step checklist surfaced inside the dashboard.",
"description": "Activation has been flat for three quarters. The new onboarding surfaces the four highest-leverage steps inline rather than on a separate page. Hypothesized to improve 7-day activation by reducing context switching.",
"hypothesis": "Surfacing the four core onboarding steps inline in the dashboard will increase 7-day activation rate by at least 3 percentage points.",
"learningGoal": "Does inline, progressive onboarding outperform a separate setup page for new accounts?",
"businessGoal": "activation",
"productArea": "onboarding",
"tags": [
"activation",
"onboarding",
"north-star"
],
"status": "decided",
"owner": {
"name": "Priya Shah",
"team": "Growth"
},
"stakeholders": [
{
"name": "Priya Shah",
"role": "pm"
},
{
"name": "Marcus Lee",
"role": "engineer"
},
{
"name": "Hana Suzuki",
"role": "data_scientist"
},
{
"name": "Jordan Patel",
"role": "designer"
}
],
"links": [
{
"type": "prd",
"title": "Onboarding redesign PRD",
"url": "https://example.com/prd-1842"
},
{
"type": "figma",
"title": "Final designs",
"url": "https://figma.com/file/onboarding-2026q2"
},
{
"type": "jira",
"title": "GROW-1842",
"url": "https://example.atlassian.net/browse/GROW-1842"
}
]
},
"design": {
"type": "ab",
"randomizationUnit": "account",
"analysisUnit": "account",
"assignmentMethod": "deterministic_hash",
"hashAttribute": "account_id",
"hashSalt": "exp_8d4a_v1",
"namespace": "growth-2026q2",
"population": "Accounts created on or after 2026-03-01 in self-serve plans.",
"targetingRules": [
{
"attribute": "plan_type",
"op": "in",
"value": [
"free",
"starter",
"pro_trial"
]
},
{
"attribute": "country",
"op": "in",
"value": [
"US",
"CA",
"GB",
"AU"
]
}
],
"exclusionRules": [
{
"attribute": "is_internal",
"op": "eq",
"value": true
},
{
"attribute": "is_bot",
"op": "eq",
"value": true
}
],
"trafficAllocation": 1,
"variantAllocation": {
"control": 0.5,
"new_onboarding": 0.5
},
"startDate": "2026-03-04T00:00:00Z",
"endDate": "2026-04-15T00:00:00Z",
"exposureDefinition": "Account_id was assigned to a variant AND signed in at least once during the experiment window.",
"triggerDefinition": "Account first reached the dashboard after assignment.",
"rampSchedule": [
{
"date": "2026-03-04",
"allocation": 0.1
},
{
"date": "2026-03-07",
"allocation": 0.5
},
{
"date": "2026-03-11",
"allocation": 1
}
],
"concurrentExperiments": [
"exp_7c12_paywall_copy"
],
"interferenceRisk": {
"level": "low",
"rationale": "Concurrent test affects pricing page only; no overlap with onboarding surface."
},
"power": 0.8,
"minimumDetectableEffect": 0.02,
"alpha": 0.05,
"multipleTestingPolicy": "metric_family",
"peekingPolicy": "fixed_horizon",
"stoppingRule": "Stop at planned end date unless guardrail metric 'support_contacts_per_account' degrades by >5% with p<0.05."
},
"variants": [
{
"id": "var_ctrl",
"key": "control",
"name": "Control — existing onboarding",
"role": "control",
"description": "The existing standalone /setup page.",
"allocation": 0.5,
"featureFlagKeys": [
"onboarding-redesign"
],
"config": {
"variant": "control"
},
"codeReferences": [
{
"type": "git_commit",
"value": "a91c4f2",
"repo": "growthbook/web"
}
]
},
{
"id": "var_new",
"key": "new_onboarding",
"name": "Treatment — inline checklist",
"role": "treatment",
"description": "Inline 4-step checklist rendered at the top of the dashboard until all steps are complete.",
"allocation": 0.5,
"featureFlagKeys": [
"onboarding-redesign"
],
"config": {
"variant": "checklist_inline"
},
"screenshots": [
{
"uri": "artifacts/treatment_checklist.png",
"caption": "Inline checklist on dashboard."
}
],
"codeReferences": [
{
"type": "git_commit",
"value": "b8f1e07",
"repo": "growthbook/web"
}
]
}
],
"metrics": [
{
"id": "activation_rate",
"name": "7-day activation rate",
"description": "Share of newly created accounts that complete all four activation events within 7 days of signup.",
"role": "primary",
"direction": "increase_is_good",
"type": "conversion",
"unit": "accounts",
"numerator": {
"description": "Accounts completing all 4 activation events within 7 days of signup."
},
"denominator": {
"description": "Accounts exposed to the experiment."
},
"aggregation": "mean",
"analysisWindow": {
"type": "conversion_window",
"days": 7
},
"dataSource": {
"warehouse": "snowflake",
"table": "analytics.fct_activation_daily"
},
"filters": [],
"owner": {
"name": "Hana Suzuki",
"team": "Data Science"
}
},
{
"id": "support_contacts_per_account",
"name": "Support contacts per account (7d)",
"description": "Average number of support contacts opened by an exposed account within 7 days.",
"role": "guardrail",
"direction": "decrease_is_good",
"type": "count",
"unit": "contacts/account",
"aggregation": "mean",
"analysisWindow": {
"type": "conversion_window",
"days": 7
},
"dataSource": {
"warehouse": "snowflake",
"table": "analytics.fct_support_contacts"
}
},
{
"id": "dashboard_load_p95_ms",
"name": "Dashboard load time (p95)",
"description": "p95 dashboard page-load latency in milliseconds.",
"role": "guardrail",
"direction": "decrease_is_good",
"type": "percentile",
"unit": "ms",
"aggregation": "percentile",
"dataSource": {
"warehouse": "snowflake",
"table": "analytics.fct_page_perf"
}
}
],
"analysis": {
"method": "frequentist",
"model": "welch_t_test",
"varianceEstimator": "naive",
"confidenceLevel": 0.95,
"alpha": 0.05,
"adjustmentMethod": "cuped",
"multipleComparisonCorrection": "benjamini_hochberg",
"segmentation": [
"country",
"plan_type"
],
"dimensionBreakdowns": [
"country",
"plan_type",
"device"
],
"missingDataHandling": "exclude_units_with_no_outcome",
"outlierHandling": "winsorize_p99_for_revenue_metrics",
"queryReferences": [
{
"type": "snowflake_query",
"id": "01a4-9f1c-77be",
"description": "Primary analysis query."
}
],
"generatedAt": "2026-04-16T09:14:00Z"
},
"results": {
"sampleSizes": {
"control": 50123,
"new_onboarding": 49987
},
"exposures": {
"control": 50123,
"new_onboarding": 49987
},
"metricResults": [
{
"metricId": "activation_rate",
"role": "primary",
"comparison": {
"baselineVariantId": "var_ctrl",
"variantId": "var_new"
},
"baselineValue": 0.421,
"variantValue": 0.447,
"absoluteDifference": 0.026,
"relativeDifference": 0.0618,
"standardError": 0.0091,
"confidenceInterval": {
"level": 0.95,
"lower": 0.008,
"upper": 0.044
},
"pValue": 0.004,
"resultStatus": "positive",
"decisionImpact": "supports_ship"
},
{
"metricId": "support_contacts_per_account",
"role": "guardrail",
"comparison": {
"baselineVariantId": "var_ctrl",
"variantId": "var_new"
},
"baselineValue": 0.182,
"variantValue": 0.178,
"absoluteDifference": -0.004,
"relativeDifference": -0.022,
"confidenceInterval": {
"level": 0.95,
"lower": -0.012,
"upper": 0.004
},
"pValue": 0.31,
"resultStatus": "neutral",
"decisionImpact": "supports_ship"
},
{
"metricId": "dashboard_load_p95_ms",
"role": "guardrail",
"comparison": {
"baselineVariantId": "var_ctrl",
"variantId": "var_new"
},
"baselineValue": 812,
"variantValue": 824,
"absoluteDifference": 12,
"relativeDifference": 0.0148,
"confidenceInterval": {
"level": 0.95,
"lower": -3,
"upper": 27
},
"pValue": 0.12,
"resultStatus": "neutral",
"decisionImpact": "supports_ship"
}
]
},
"scorecard": {
"summary": "Treatment lifted 7-day activation by 2.6 points (+6.2% relative, p=0.004) with no guardrail regressions.",
"primaryOutcome": {
"metricId": "activation_rate",
"result": "win",
"relativeDifference": 0.0618,
"confidenceInterval": {
"level": 0.95,
"lower": 0.008,
"upper": 0.044
}
},
"guardrailOutcomes": [
{
"metricId": "support_contacts_per_account",
"status": "pass"
},
{
"metricId": "dashboard_load_p95_ms",
"status": "pass"
}
],
"qualityStatus": "valid",
"overallResult": "win",
"recommendedAction": "ship",
"keyFindings": [
"Activation rate lifted from 42.1% to 44.7% (+2.6 pp).",
"No guardrail regressions on support contacts or dashboard latency.",
"Effect was consistent across US, CA, GB, AU and across plan tiers."
],
"risks": [
"Novelty effect cannot be ruled out for the first 7 days post-launch.",
"Effect on long-tail retention metrics is not yet known."
],
"charts": [
{
"uri": "artifacts/primary_metric.png",
"caption": "Activation rate over time by variant."
}
]
},
"decision": {
"status": "decided",
"outcome": "ship",
"rationale": "Primary metric met MDE with no guardrail regressions. Recommend full rollout with a 90-day retention follow-up.",
"decidedBy": {
"name": "Priya Shah",
"role": "pm"
},
"decidedAt": "2026-04-18T13:50:00Z",
"followUps": [
{
"type": "retention_followup",
"title": "90-day retention check",
"dueDate": "2026-07-18"
}
],
"rolloutPlan": {
"type": "gradual_ramp",
"schedule": [
{
"date": "2026-04-22",
"allocation": 0.25
},
{
"date": "2026-04-29",
"allocation": 1
}
]
},
"productChanges": "Inline onboarding checklist is now the default for all self-serve accounts."
},
"qualityChecks": [
{
"checkType": "sample_ratio_mismatch",
"status": "pass",
"severity": "high",
"observed": {
"control": 50123,
"treatment": 49987
},
"expected": {
"control": 0.5,
"treatment": 0.5
},
"pValue": 0.61,
"message": "No evidence of sample ratio mismatch."
},
{
"checkType": "exposure_health",
"status": "pass",
"severity": "medium",
"message": "Exposure logging consistent across variants throughout the run."
},
{
"checkType": "data_freshness",
"status": "pass",
"severity": "low",
"message": "All warehouse tables refreshed within 2 hours of analysis time."
}
],
"artifacts": [
{
"type": "chart",
"title": "Primary metric over time",
"uri": "artifacts/primary_metric.png",
"mimeType": "image/png"
},
{
"type": "sql",
"title": "Primary analysis query",
"uri": "artifacts/analysis.sql",
"mimeType": "text/sql"
},
{
"type": "html_report",
"title": "Full scorecard",
"uri": "artifacts/scorecard.html",
"mimeType": "text/html"
}
],
"provenance": {
"createdBy": {
"system": "GrowthBook",
"user": "priya@example.com"
},
"exportedBy": {
"system": "GrowthBook",
"user": "priya@example.com"
},
"analysisGeneratedBy": "growthbook-stats v3.4.1",
"dataSources": [
{
"warehouse": "snowflake",
"schema": "analytics",
"tables": [
"fct_activation_daily",
"fct_support_contacts",
"fct_page_perf"
]
}
],
"queryIds": [
"01a4-9f1c-77be"
],
"codeVersion": "growthbook/web@b8f1e07",
"metricDefinitionVersion": "v2026.04.10"
},
"extensions": {
"growthbook:engineSettings": {
"statsEngine": "frequentist",
"regressionAdjustmentEnabled": true
}
}
}