Skip to content

microsoft/power-platform-skills / plugins/power-pages/skills/migrate-datamodel

migrate-datamodel

Verified

This skill should be used when the user asks to "migrate to enhanced data model", "migrate from standard to enhanced", "switch to EDM", "migrate SDM to EDM", "upgrade data model", "migrate site data model", or wants to migrate an existing Power Pages site from the Standard Data Model (SDM) to the Enhanced Data Model (EDM) using PAC CLI.

scripts/snapshot-site.js

View on GitHub
#!/usr/bin/env node
/**
 * snapshot-site.js
 *
 * Walks a `pac pages download` site tree and builds a JSON snapshot of every
 * artifact it knows about. Pre-migration (modelVersion 1) and post-migration
 * (modelVersion 2) snapshots have the same shape, so they can be diffed by
 * diff-snapshots.js.
 *
 * USAGE
 *   node snapshot-site.js --site-root <path> --output-dir <path> --label sdm|edm
 *
 * OUTPUT
 *   <output-dir>/<label>-snapshot.json
 *
 * Identity fields per artifact are deliberately narrow (name + language +
 * statecode) so the diff is robust against harmless YAML reordering and
 * whitespace differences. Pass --with-content-hash to also include SHA-256 of
 * the companion content file (webpage copy HTML, snippet value HTML, web
 * template source, …) when one exists — stricter but can false-positive on
 * encoding/line-ending shifts.
 */

const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { parseSimpleYaml } = require('../../../scripts/lib/powerpages-config');
const { parseTopLevelSequence } = require('../../../scripts/lib/non-spa-metadata');
const { requirePowerPagesSiteRoot } = require('../../../scripts/lib/validation-helpers');

function parseArgs(argv) {
  const out = {};
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a.startsWith('--')) {
      const k = a.slice(2);
      const next = argv[i + 1];
      if (next === undefined || next.startsWith('--')) {
        out[k] = true;
      } else {
        out[k] = next;
        i += 1;
      }
    }
  }
  return out;
}

function safeReaddir(p) {
  try {
    return fs.readdirSync(p, { withFileTypes: true });
  } catch (e) {
    if (e.code === 'ENOENT') return [];
    throw new Error(`Could not read snapshot directory ${p}: ${e.message}`);
  }
}

function findFilesBySuffix(dir, suffix, recursive = false) {
  return safeReaddir(dir)
    .flatMap((e) => {
      const file = path.join(dir, e.name);
      if (recursive && e.isDirectory()) return findFilesBySuffix(file, suffix, true);
      return e.isFile() && e.name.endsWith(suffix) ? [file] : [];
    });
}

// Aggregate PAC records use "adx_statecode: 1"; per-record exports can use
// "statecode: 1" (and "name" instead of "adx_name"). Keep explicit zero/false.
function recordField(record, key) {
  const alternate = key.startsWith('adx_') ? key.slice(4) : `adx_${key}`;
  const value = record[key] ?? record[alternate] ?? null;
  return Array.isArray(value) ? value.slice().sort() : value;
}

function stateCode(record) {
  const value = recordField(record, 'statecode');
  return typeof value === 'number' ? value : null;
}

function mergeScans(...scans) {
  const inventory = scans.flatMap((scan) => scan.inventory);
  return { count: inventory.length, inventory, parseErrors: scans.flatMap((scan) => scan.parseErrors) };
}

function safeParseYaml(filePath, errors) {
  try {
    return parseSimpleYaml(fs.readFileSync(filePath, 'utf8'), filePath);
  } catch (e) {
    errors.push({ filePath, message: e.message });
    return null;
  }
}

function sha256OfFileIfExists(p) {
  if (!p) return null;
  try {
    return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex').slice(0, 16);
  } catch (error) {
    if (error.code === 'ENOENT') return null;
    throw new Error(`Could not hash snapshot content ${p}: ${error.message}`);
  }
}

// Extract a language tag from a filename like "About-Footer.en-US.contentsnippet.yml".
// Returns null if the segment doesn't match a BCP-47-ish pattern.
function extractLanguageTag(filename, suffix) {
  const base = filename.slice(0, -suffix.length);
  const segments = base.split('.');
  if (segments.length < 2) return null;
  const last = segments[segments.length - 1];
  return /^[a-z]{2,3}(-[A-Z]{2,4})?$/i.test(last) ? last : null;
}

// Reuse strict aggregate parsing so malformed fields cannot silently disappear
// from a migration baseline. It also handles quoted and multiline YAML scalars.
function parseCollectionFile(filePath, errors) {
  try {
    // Read directly: existsSync can report false for inaccessible files too,
    // but only ENOENT is an optional missing collection, not incomplete evidence.
    return parseTopLevelSequence(fs.readFileSync(filePath, 'utf8'), filePath);
  } catch (e) {
    if (e.code === 'ENOENT') return [];
    errors.push({ filePath, message: e.message });
    return [];
  }
}

// ── Per-artifact scanners ──────────────────────────────────────────────────

function scanWebPages(siteRoot, withContentHash) {
  const errors = [];
  const inventory = [];
  const root = path.join(siteRoot, 'web-pages');

  for (const slug of safeReaddir(root)) {
    if (!slug.isDirectory()) continue;
    const slugDir = path.join(root, slug.name);

    // Parent page: <Name>.webpage.yml at <slug>/ (not in content-pages/)
    for (const f of findFilesBySuffix(slugDir, '.webpage.yml')) {
      const rec = safeParseYaml(f, errors);
      if (!rec) continue;
      inventory.push({
        kind: 'parent',
        slug: slug.name,
        name: recordField(rec, 'adx_name'),
        partialUrl: recordField(rec, 'adx_partialurl'),
        isRoot: [true, 'true'].includes(recordField(rec, 'adx_isroot')),
        stateCode: stateCode(rec),
        contentHash: withContentHash ? sha256OfFileIfExists(f.replace('.webpage.yml', '.webpage.copy.html')) : undefined,
      });
    }

    // PAC layouts include content-pages/<Name>.<lang>.webpage.yml and
    // content-pages/<locale>/<Name>.<lang>.webpage.yml.
    const contentPagesDir = path.join(slugDir, 'content-pages');
    for (const f of findFilesBySuffix(contentPagesDir, '.webpage.yml', true)) {
      const rec = safeParseYaml(f, errors);
      if (!rec) continue;
      inventory.push({
        kind: 'content',
        slug: slug.name,
        name: recordField(rec, 'adx_name'),
        partialUrl: recordField(rec, 'adx_partialurl'),
        language: extractLanguageTag(path.basename(f), '.webpage.yml'),
        stateCode: stateCode(rec),
        contentHash: withContentHash ? sha256OfFileIfExists(f.replace('.webpage.yml', '.webpage.copy.html')) : undefined,
      });
    }
  }
  return { count: inventory.length, inventory, parseErrors: errors };
}

function scanContentSnippets(siteRoot, withContentHash) {
  const errors = [];
  const inventory = [];
  const root = path.join(siteRoot, 'content-snippets');
  for (const slug of safeReaddir(root)) {
    if (!slug.isDirectory()) continue;
    const dir = path.join(root, slug.name);
    for (const f of findFilesBySuffix(dir, '.contentsnippet.yml')) {
      const rec = safeParseYaml(f, errors);
      if (!rec) continue;
      inventory.push({
        slug: slug.name,
        name: recordField(rec, 'adx_name'),
        displayName: recordField(rec, 'adx_display_name'),
        language: extractLanguageTag(path.basename(f), '.contentsnippet.yml'),
        stateCode: stateCode(rec),
        contentHash: withContentHash ? sha256OfFileIfExists(f.replace('.contentsnippet.yml', '.contentsnippet.value.html')) : undefined,
      });
    }
  }
  return { count: inventory.length, inventory, parseErrors: errors };
}

function scanWebLinkSets(siteRoot, folder) {
  const errors = [];
  const inventory = [];
  const root = path.join(siteRoot, folder);
  for (const slug of safeReaddir(root)) {
    if (!slug.isDirectory()) continue;
    const dir = path.join(root, slug.name);
    for (const f of findFilesBySuffix(dir, '.weblinkset.yml')) {
      // Skip the sibling weblink-collection file (ends with .weblinkset.weblink.yml)
      if (f.endsWith('.weblinkset.weblink.yml')) continue;
      const rec = safeParseYaml(f, errors);
      if (!rec) continue;
      const language = extractLanguageTag(path.basename(f), '.weblinkset.yml');

      // Read sibling weblinks file (collection format)
      const linksFile = f.replace('.weblinkset.yml', '.weblinkset.weblink.yml');
      const linkRecs = parseCollectionFile(linksFile, errors);
      const links = linkRecs
        .map((r) => ({
          name: recordField(r, 'adx_name'),
          displayOrder: recordField(r, 'adx_displayorder'),
          pageId: recordField(r, 'adx_pageid'),
          externalUrl: recordField(r, 'adx_externalurl'),
          stateCode: stateCode(r),
        }))
        .sort((a, b) => (a.displayOrder ?? 999) - (b.displayOrder ?? 999) || String(a.name).localeCompare(String(b.name)));

      inventory.push({
        slug: slug.name,
        name: recordField(rec, 'adx_name'),
        language,
        stateCode: stateCode(rec),
        linkCount: links.filter((link) => (link.stateCode ?? 0) === 0).length,
        linkNames: links.filter((link) => (link.stateCode ?? 0) === 0).map((l) => l.name),
        links,
      });
    }
  }
  return { count: inventory.length, inventory, parseErrors: errors };
}

// Derive a flat per-link inventory from the per-set link data captured by
// scanWebLinkSets. Each record represents a single navigation link with its
// parent set's slug for context.
function deriveWebLinks(linkSetsResult) {
  const inventory = [];
  for (const set of linkSetsResult.inventory || []) {
    for (const link of set.links || []) {
      inventory.push({
        ...link,
        slug: set.slug,
        parentSet: set.name,
        language: set.language,
      });
    }
  }
  return { count: inventory.length, inventory, parseErrors: [] };
}

function scanWebTemplates(siteRoot, withContentHash) {
  const errors = [];
  const inventory = [];
  const root = path.join(siteRoot, 'web-templates');
  for (const slug of safeReaddir(root)) {
    if (!slug.isDirectory()) continue;
    const dir = path.join(root, slug.name);
    for (const f of findFilesBySuffix(dir, '.webtemplate.yml')) {
      const rec = safeParseYaml(f, errors);
      if (!rec) continue;
      inventory.push({
        slug: slug.name,
        name: recordField(rec, 'adx_name'),
        stateCode: stateCode(rec),
        contentHash: withContentHash ? sha256OfFileIfExists(f.replace('.webtemplate.yml', '.webtemplate.source.html')) : undefined,
      });
    }
  }
  return { count: inventory.length, inventory, parseErrors: errors };
}

// Flat folder: <root>/<Name>.<artifactKind>.yml (no per-name subfolder)
function scanFlatFolder(siteRoot, folder, suffix, identity) {
  const errors = [];
  const inventory = [];
  const dir = path.join(siteRoot, folder);
  for (const f of findFilesBySuffix(dir, suffix)) {
    const rec = safeParseYaml(f, errors);
    if (!rec) continue;
    const item = {};
    for (const [outKey, ymlKey] of Object.entries(identity)) {
      item[outKey] = recordField(rec, ymlKey);
    }
    item.stateCode = stateCode(rec);
    inventory.push(item);
  }
  return { count: inventory.length, inventory, parseErrors: errors };
}

// Per-slug subfolder: <root>/<slug>/<slug>.<artifactKind>.yml
function scanPerSlugFolder(siteRoot, folder, suffix, identity) {
  const errors = [];
  const inventory = [];
  const root = path.join(siteRoot, folder);
  for (const slug of safeReaddir(root)) {
    if (!slug.isDirectory()) continue;
    const dir = path.join(root, slug.name);
    for (const f of findFilesBySuffix(dir, suffix)) {
      const rec = safeParseYaml(f, errors);
      if (!rec) continue;
      const item = { slug: slug.name };
      for (const [outKey, ymlKey] of Object.entries(identity)) {
        item[outKey] = recordField(rec, ymlKey);
      }
      item.stateCode = stateCode(rec);
      inventory.push(item);
    }
  }
  return { count: inventory.length, inventory, parseErrors: errors };
}

// Top-level collection file (an array of records). PAC names: sitesetting.yml,
// webrole.yml, sitemarker.yml, websiteaccess.yml, publishingstate.yml, etc.
function scanTopLevelCollection(siteRoot, file, identity) {
  const errors = [];
  const inventory = [];
  const full = path.join(siteRoot, file);
  for (const rec of parseCollectionFile(full, errors)) {
    const item = {};
    for (const [outKey, ymlKey] of Object.entries(identity)) {
      item[outKey] = recordField(rec, ymlKey);
    }
    item.stateCode = stateCode(rec);
    inventory.push(item);
  }
  return { count: inventory.length, inventory, parseErrors: errors };
}

function scanAdvancedFormSteps(siteRoot, forms) {
  const errors = [];
  const inventory = [];
  for (const form of forms.inventory) {
    const root = path.join(siteRoot, 'advanced-forms', form.slug, 'advanced-form-steps');
    for (const file of findFilesBySuffix(root, '.advancedformstep.yml', true)) {
      const rec = safeParseYaml(file, errors);
      if (!rec) continue;
      inventory.push({
        name: recordField(rec, 'adx_name'),
        parentForm: form.name,
        formId: recordField(rec, 'adx_webform'),
        type: recordField(rec, 'adx_type'),
        targetEntity: recordField(rec, 'adx_targetentitylogicalname'),
        nextStepId: recordField(rec, 'adx_nextstep'),
        stateCode: stateCode(rec),
      });
    }
  }
  return { count: inventory.length, inventory, parseErrors: errors };
}

// ── Canonicalize inventory for stable diffs ────────────────────────────────

function sortInventory(arr, ...keys) {
  return arr.slice().sort((a, b) => {
    for (const k of keys) {
      const av = String(a[k] ?? '');
      const bv = String(b[k] ?? '');
      if (av < bv) return -1;
      if (av > bv) return 1;
    }
    return 0;
  });
}

// ── Main ───────────────────────────────────────────────────────────────────

function main() {
  const args = parseArgs(process.argv.slice(2));
  let siteRoot = args['site-root'];
  const outputDir = args['output-dir'];
  const label = args['label'];
  const withContentHash = !!args['with-content-hash'];

  if (!siteRoot) { console.error('--site-root is required'); process.exit(1); }
  if (!outputDir) { console.error('--output-dir is required'); process.exit(1); }
  if (!label || !['sdm', 'edm'].includes(label)) {
    console.error('--label must be "sdm" or "edm"'); process.exit(1);
  }
  siteRoot = requirePowerPagesSiteRoot(siteRoot);
  const webLinkSets = mergeScans(
    scanWebLinkSets(siteRoot, 'web-link-sets'),
    scanWebLinkSets(siteRoot, 'weblink-sets'));
  const siteSettings = mergeScans(
    scanTopLevelCollection(siteRoot, 'sitesetting.yml', { name: 'adx_name', value: 'adx_value' }),
    scanFlatFolder(siteRoot, 'site-settings', '.sitesetting.yml', { name: 'adx_name', value: 'adx_value' }));
  const ruleFields = {
    name: 'adx_name', pageId: 'adx_webpageid', right: 'adx_right', scope: 'adx_scope',
    webRoleIds: 'adx_webpageaccesscontrolrule_webrole',
  };
  const webpageRules = mergeScans(
    scanTopLevelCollection(siteRoot, 'webpagerule.yml', ruleFields),
    scanFlatFolder(siteRoot, 'webpage-rules', '.webpagerule.yml', ruleFields));
  const advancedForms = scanPerSlugFolder(siteRoot, 'advanced-forms', '.advancedform.yml',
    { name: 'adx_name' });

  const categories = {
    webPages: scanWebPages(siteRoot, withContentHash),
    contentSnippets: scanContentSnippets(siteRoot, withContentHash),
    webLinkSets,
    // webLinks is derived from the per-set link inventory captured by
    // scanWebLinkSets above. It exposes each individual link as its own
    // record so the Navigation group in the live report shows the actual
    // navigation items, not just the menu containers.
    webLinks: deriveWebLinks(webLinkSets),
    webTemplates: scanWebTemplates(siteRoot, withContentHash),
    webFiles: scanFlatFolder(siteRoot, 'web-files', '.webfile.yml',
      { name: 'adx_name', partialUrl: 'adx_partialurl' }),
    pageTemplates: scanFlatFolder(siteRoot, 'page-templates', '.pagetemplate.yml',
      { name: 'adx_name' }),
    tablePermissions: scanFlatFolder(siteRoot, 'table-permissions', '.tablepermission.yml',
      { name: 'adx_entityname', permissionName: 'adx_name' }),
    basicForms: scanPerSlugFolder(siteRoot, 'basic-forms', '.basicform.yml',
      { name: 'adx_name' }),
    advancedForms,
    advancedFormSteps: scanAdvancedFormSteps(siteRoot, advancedForms),
    lists: mergeScans(
      scanFlatFolder(siteRoot, 'lists', '.list.yml', { name: 'adx_name' }),
      scanPerSlugFolder(siteRoot, 'lists', '.entitylist.yml', { name: 'adx_name' })),
    polls: scanPerSlugFolder(siteRoot, 'polls', '.poll.yml',
      { name: 'adx_name' }),
    pollPlacements: scanPerSlugFolder(siteRoot, 'poll-placements', '.pollplacement.yml',
      { name: 'adx_name' }),
    cloudFlowConsumers: mergeScans(
      scanFlatFolder(siteRoot, 'cloud-flow-consumer', '.cloudflowconsumer.yml', { name: 'adx_name' }),
      scanPerSlugFolder(siteRoot, 'cloud-flow-consumer', '.cloudflowconsumer.yml', { name: 'adx_name' })),
    columnPermissionProfiles: scanPerSlugFolder(siteRoot, 'column-permission-profiles', '.columnpermissionprofile.yml',
      { name: 'adx_name' }),
    siteSettings,
    webRoles: mergeScans(
      scanTopLevelCollection(siteRoot, 'webrole.yml', { name: 'adx_name' }),
      scanFlatFolder(siteRoot, 'web-roles', '.webrole.yml', { name: 'adx_name' })),
    siteMarkers: scanTopLevelCollection(siteRoot, 'sitemarker.yml',
      { name: 'adx_name' }),
    websiteAccesses: scanTopLevelCollection(siteRoot, 'websiteaccess.yml',
      { name: 'adx_name' }),
    publishingStates: scanTopLevelCollection(siteRoot, 'publishingstate.yml',
      { name: 'adx_name' }),
    webpageRules,
    websiteBindings: scanTopLevelCollection(siteRoot, 'websitebinding.yml',
      { name: 'adx_name' }),
    websiteLanguages: scanTopLevelCollection(siteRoot, 'websitelanguage.yml',
      { name: 'adx_name' }),
    ads: scanTopLevelCollection(siteRoot, 'ad.yml',
      { name: 'adx_name' }),
    adPlacements: scanTopLevelCollection(siteRoot, 'adplacement.yml',
      { name: 'adx_name' }),
    tags: scanTopLevelCollection(siteRoot, 'tag.yml',
      { name: 'adx_name' }),
    urlHistory: scanTopLevelCollection(siteRoot, 'urlhistory.yml',
      { name: 'adx_url_logicalname' }),
  };

  // Canonical sort keys per category.
  const sortKeys = {
    webPages: ['kind', 'slug', 'partialUrl', 'language'],
    contentSnippets: ['slug', 'language', 'name'],
    webLinkSets: ['slug', 'language', 'name'],
    webLinks: ['slug', 'language', 'name'],
    webTemplates: ['slug', 'name'],
    webFiles: ['name', 'partialUrl'],
    pageTemplates: ['name'],
    tablePermissions: ['name', 'permissionName'],
    basicForms: ['slug', 'name'],
    advancedForms: ['slug', 'name'],
    advancedFormSteps: ['parentForm', 'name'],
    lists: ['slug', 'name'],
    polls: ['slug', 'name'],
    pollPlacements: ['slug', 'name'],
    cloudFlowConsumers: ['slug', 'name'],
    columnPermissionProfiles: ['slug', 'name'],
    siteSettings: ['name'],
    webRoles: ['name'],
    siteMarkers: ['name'],
    websiteAccesses: ['name'],
    publishingStates: ['name'],
    webpageRules: ['name'],
    websiteBindings: ['name'],
    websiteLanguages: ['name'],
    ads: ['name'],
    adPlacements: ['name'],
    tags: ['name'],
    urlHistory: ['name'],
  };
  for (const [cat, keys] of Object.entries(sortKeys)) {
    if (categories[cat]) {
      categories[cat].inventory = sortInventory(categories[cat].inventory, ...keys);
    }
  }

  const counts = {};
  for (const [cat, data] of Object.entries(categories)) counts[cat] = data.count;

  const snapshot = {
    schemaVersion: 1,
    label,
    siteRoot: path.resolve(siteRoot),
    capturedAt: new Date().toISOString(),
    withContentHash,
    counts,
    inventory: Object.fromEntries(
      Object.entries(categories).map(([cat, data]) => [cat, data.inventory]),
    ),
    parseErrors: Object.fromEntries(
      Object.entries(categories)
        .map(([cat, data]) => [cat, data.parseErrors])
        .filter(([, errs]) => errs.length > 0),
    ),
  };

  fs.mkdirSync(outputDir, { recursive: true });
  const outPath = path.join(outputDir, `${label}-snapshot.json`);
  fs.writeFileSync(outPath, JSON.stringify(snapshot, null, 2) + '\n', 'utf8');

  console.log(`✓ Wrote ${outPath}`);
  const totalErrors = Object.values(snapshot.parseErrors).reduce((n, errs) => n + errs.length, 0);
  console.log('');
  console.log(`  Category                    Count`);
  console.log(`  ──────────────────────────  ─────`);
  for (const [cat, n] of Object.entries(counts)) {
    if (n === 0) continue; // hide noisy zeros for sites that don't use every artifact
    console.log(`  ${cat.padEnd(26)}  ${String(n).padStart(5)}`);
  }
  if (totalErrors > 0) {
    console.log('');
    console.log(`  ⚠ ${totalErrors} YAML parse errors (see parseErrors in snapshot file)`);
  }
}

if (require.main === module) main();

module.exports = { safeReaddir };
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall