#!/usr/bin/env python3
"""Turn CRM definition exports into the JSON fragments that audit_crm.py reads.

Usage (from a private working folder; SKILL_DIR is this skill's folder; S="$SKILL_DIR/scripts"):
  python3 -I "$S/normalize_exports.py" hubspot-properties <csv...> [--object contacts]
      [--map "fill_rate=Fill rate (%)"] --out audit-work/fields.json
  python3 -I "$S/normalize_exports.py" hubspot-flows <flows.json...> [--credit-action-ids 0-1,0-2]
      --out audit-work/automations.json
  python3 -I "$S/normalize_exports.py" salesforce-describe <describe.json...>
      [--descriptions fielddefinition-opportunity.csv] --out audit-work/fields.json
  python3 -I "$S/normalize_exports.py" salesforce-flows <FlowDefinitionView.csv> --out audit-work/automations.json
  python3 -I "$S/normalize_exports.py" attio-attributes <attributes.json...> [--object companies]
      [--options options.json] --out audit-work/fields.json
  python3 -I "$S/normalize_exports.py" generic-fields <fields.csv...> --out audit-work/fields.json

Reads definitions only: field settings and workflow settings, never record values. Makes no network
calls and needs no third-party modules. The header mapping and anything it could not map are printed
to stderr so a person can check them. Header names for vendor exports change: when a required column
is not recognised, pass --map key=Header or ask the user for the header row only. HubSpot workflow
actions the script cannot classify are listed per workflow in unknown_action_types, never dropped. A
HubSpot flow without an actions list (a summary from the v4 list endpoint or `workflows list`) is kept
with writes_known, actions_known and refs_known false and named in a note, so no rule passes on it.
Filters on events and associated objects are counted in other_object_refs, never resolved against the
enrolled object's fields. A Create record action's properties and an Edit record on an associated record
are writes on that other object.

Exit code: 0 on success, 2 on bad input or when a required column cannot be mapped.
"""
import argparse
import csv
import json
import os
import re
import sys

KNOWN_OBJECTS = {
    "contact": "contacts", "contacts": "contacts", "company": "companies", "companies": "companies",
    "deal": "deals", "deals": "deals", "ticket": "tickets", "tickets": "tickets", "lead": "leads",
    "leads": "leads", "people": "people", "person": "people", "account": "accounts", "accounts": "accounts",
    "opportunity": "opportunities", "opportunities": "opportunities", "workspace": "workspaces",
    "workspaces": "workspaces",
}

# HubSpot workflow action type IDs checked on 2026-11-01 (workflows v4 action reference).
HUBSPOT_ACTIONS = {
    "0-5": "edit_record", "0-11": "rotate_owner", "0-14": "create_record", "0-15": "go_to_workflow",
    "0-4": "send_email", "0-31": "set_marketing_contact_status",
    "0-43347357": "manage_communication_subscriptions", "0-18224765": "delete_contact",
    "0-46510720": "enroll_in_sequence", "0-177946906": "enrich_record",
    "0-172351286": "data_agent_custom_prompt", "0-216647524": "data_agent_research",
    "0-201091202": "data_agent_fill_smart_property", "0-219292676": "enroll_in_prospecting_agent",
}
# Core actions that write no CRM property, delete nothing, send nothing to contacts and change no consent:
# delays (0-1, 0-29, 0-35), internal notifications (0-8, 0-9, 0-23), tasks (0-3), notes (0-169425243),
# static lists (0-63809083, 0-63863438), associations and association labels (0-63189541, 0-73444249,
# 0-61139484, 0-61139476), ads audiences (0-30), sequence and prospecting agent unenrolment (0-4702372,
# 0-217709844) and conversation owner assignment (0-44475148). 0-13 is the older static list action: the
# workflows guide's GET example shows it (fields operation and list_id), the action reference does not.
# Every other ID that is not in HUBSPOT_ACTIONS is listed per workflow as unclassified, including actions
# that text contacts (SMS, WhatsApp, surveys), write to Salesforce (campaign, task) or track intent signals.
HUBSPOT_NEUTRAL_ACTIONS = {
    "0-1", "0-29", "0-35", "0-8", "0-9", "0-23", "0-3", "0-13", "0-169425243", "0-63809083", "0-63863438",
    "0-63189541", "0-73444249", "0-61139484", "0-61139476", "0-30", "0-4702372", "0-217709844", "0-44475148",
}
CONSENT_ACTIONS = {"0-31", "0-43347357"}
DESTRUCTIVE_ACTIONS = {"0-18224765"}
# The three Data Agent actions (beta) and prospecting agent enrolment use HubSpot credits. Data enrichment does
# not (HubSpot's data enrichment guide), so Enrich record is not here. Other AI actions (0-195318603,
# 0-207702619, 0-218870680) need credits too but can write properties, so they stay unclassified: pass them
# with --credit-action-ids.
CREDIT_ACTIONS = {"0-172351286", "0-216647524", "0-201091202", "0-219292676"}
SEQUENCE_ACTIONS = {"0-46510720"}
HUBSPOT_OBJECT_TYPES = {"0-1": "contacts", "0-2": "companies", "0-3": "deals", "0-5": "tickets"}
# The Property value changed enrolment event. The action reference does not show how it names the property,
# so a flow that re-enrols on it is left for a person to read (AUTO-05).
PROPERTY_CHANGE_EVENT = "4-655002"
# Branch types whose filters read the enrolled object. Anything else (an event branch such as UNIFIED_EVENTS,
# an association branch) filters on another record's properties.
OBJECT_BRANCH_TYPES = {"", "OR", "AND", "NOT_ALL", "NOT_ANY"}

# Canonical key -> header aliases (lowercase, punctuation removed). Exact matches win over prefixes.
HUBSPOT_PROPERTY_ALIASES = [
    ("name", ["internal name", "internal value", "property internal name", "api name", "internal property name"]),
    ("label", ["name", "label", "property name", "property label", "display name"]),
    ("field_type", ["field type", "fieldtype", "input type", "control type"]),
    ("type", ["type", "property type", "data type"]),
    ("description", ["description", "property description"]),
    ("group", ["group name", "group", "property group"]),
    ("options", ["options", "option values", "dropdown options", "values"]),
    ("created_by", ["created by", "creator"]),
    ("created_at", ["created", "created at", "created date", "create date", "date created"]),
    ("usages", ["usages", "used in", "usage", "number of usages", "usage count"]),
    ("fill_rate", ["fill rate", "fill", "percent filled", "fill percentage"]),
    ("last_updated", ["last updated", "last updated time", "updated", "last modified"]),
    ("update_source", ["update source", "last update source", "source"]),
    ("object", ["object", "object type", "record type"]),
    ("custom", ["custom", "is custom", "custom property"]),
    ("vendor_defined", ["hubspot defined", "default property", "hubspot default", "default"]),
]
REQUIRED = {"hubspot-properties": ["name"], "generic-fields": ["object", "name", "type"]}

# A small list of HubSpot default properties, used only when the export has no custom flag.
HUBSPOT_DEFAULT_NAMES = {
    "email", "firstname", "lastname", "phone", "mobilephone", "company", "website", "jobtitle",
    "lifecyclestage", "hubspot_owner_id", "country", "city", "state", "zip", "address", "createdate",
    "lastmodifieddate", "closedate", "dealname", "amount", "dealstage", "pipeline", "dealtype", "name",
    "domain", "industry", "numberofemployees", "annualrevenue", "description", "notes_last_updated",
    "closed_lost_reason", "closed_won_reason", "subject", "content", "hs_pipeline", "hs_pipeline_stage",
}


def fail(msg):
    print("normalize_exports: " + msg, file=sys.stderr)
    sys.exit(2)


def note(msg):
    print("  " + msg, file=sys.stderr)


def norm_header(h):
    h = (h or "").replace("\ufeff", "").strip().lower()
    h = re.sub(r"[^a-z0-9]+", " ", h)
    return re.sub(r"\s+", " ", h).strip()


def map_headers(headers, aliases, overrides):
    mapping = {}
    used = set()
    normed = {h: norm_header(h) for h in headers}
    for key, header in overrides.items():
        if header not in headers:
            fail("--map %s=%s: no column named %r (columns: %s)" % (key, header, header, ", ".join(headers)))
        mapping[key] = header
        used.add(header)
    for key, names in aliases:  # exact matches first
        if key in mapping:
            continue
        for h in headers:
            if h not in used and normed[h] in names:
                mapping[key] = h
                used.add(h)
                break
    for key, names in aliases:  # then headers that start with an alias
        if key in mapping:
            continue
        for alias in sorted(names, key=len, reverse=True):
            hit = next((h for h in headers if h not in used and re.match(re.escape(alias) + r"\b", normed[h])), None)
            if hit:
                mapping[key] = hit
                used.add(hit)
                break
    return mapping, [h for h in headers if h not in used]


def parse_overrides(pairs):
    out = {}
    for p in pairs or []:
        if "=" not in p:
            fail("--map expects key=Header, got %r" % p)
        k, v = p.split("=", 1)
        out[k.strip()] = v.strip()
    return out


def object_from(text):
    tokens = re.split(r"[^a-z]+", (text or "").lower())
    for t in tokens:
        if t in KNOWN_OBJECTS:
            return KNOWN_OBJECTS[t]
    return None


def norm_object(value):
    v = (value or "").strip().lower()
    return KNOWN_OBJECTS.get(v, v or None)


def type_family(*values):
    s = " ".join(str(v).lower() for v in values if v)
    if not s:
        return "other"
    if any(t in s for t in ("booleancheckbox", "single checkbox", "boolean", "bool")) or s.strip() == "checkbox":
        return "bool"
    if any(t in s for t in ("datetime", "timestamp", "date")):
        return "date"
    if any(t in s for t in ("enumeration", "select", "radio", "checkbox", "picklist", "status", "dropdown")):
        return "enum"
    if any(t in s for t in ("phone", "email", "url", "domain")):
        return "text"
    if any(t in s for t in ("number", "currency", "percent", "double", "int", "long", "rating", "calculation", "score")):
        return "number"
    if any(t in s for t in ("owner", "reference", "lookup", "masterdetail")) or s.strip() in ("id", "reference"):
        return "reference"
    if any(t in s for t in ("string", "text", "phone", "email", "url", "html", "domain", "name", "location")):
        return "text"
    return "other"


def parse_bool(v):
    if v is None:
        return None
    s = str(v).strip().lower()
    if s in ("true", "yes", "y", "1", "custom"):
        return True
    if s in ("false", "no", "n", "0", "default", "standard"):
        return False
    return None


def parse_int(v):
    m = re.search(r"-?\d+", str(v or "").replace(",", ""))
    return int(m.group(0)) if m else None


def parse_options(text):
    out = []
    for raw in re.split(r";|\r?\n", text or ""):
        if not raw.strip():
            continue
        m = re.match(r"^(.*\S)\s+\(([^()]*)\)\s*$", raw.strip())
        label, value = (m.group(1), m.group(2)) if m else (raw.strip(), raw.strip())
        out.append({"label": label, "value": value.strip(), "raw": raw.lstrip(" ")})
    return out


def fill_rate_parser(values):
    """Values with % are percents; with no % sign, all at or below 1 means fractions, else percents."""
    nums = []
    pct = False
    for v in values:
        s = str(v or "").strip()
        if not s:
            continue
        if "%" in s:
            pct = True
        try:
            nums.append(float(s.replace("%", "").replace(",", "")))
        except ValueError:
            pass
    fractions = (not pct) and nums and all(0 <= n <= 1 for n in nums)

    def parse(v):
        s = str(v or "").strip().replace(",", "")
        if not s:
            return None
        try:
            n = float(s.replace("%", ""))
        except ValueError:
            return None
        return round(n if fractions else n / 100.0, 4)
    return parse, ("fractions" if fractions else "percent")


def read_csv(path):
    try:
        with open(path, newline="", encoding="utf-8-sig") as f:
            rows = list(csv.DictReader(f))
            return rows, (rows[0].keys() if rows else [])
    except (OSError, csv.Error, UnicodeDecodeError) as e:
        fail("cannot read %s: %s" % (path, e))


def read_json(path):
    try:
        with open(path, encoding="utf-8") as f:
            return json.load(f)
    except (OSError, ValueError) as e:
        fail("cannot read %s: %s" % (path, e))


def hubspot_properties(files, args):
    fields = []
    maps = {}
    overrides = parse_overrides(args.map)
    for path in files:
        rows, headers = read_csv(path)
        headers = list(headers)
        mapping, unmapped = map_headers(headers, HUBSPOT_PROPERTY_ALIASES, overrides)
        maps[os.path.basename(path)] = mapping
        print("%s: header mapping" % os.path.basename(path), file=sys.stderr)
        for k, h in mapping.items():
            note("%s <- %r" % (k, h))
        if unmapped:
            note("not used: " + ", ".join(repr(h) for h in unmapped))
        for req in REQUIRED["hubspot-properties"]:
            if req not in mapping:
                fail("%s: no column maps to %r. Pass --map %s=<header> (columns: %s)" % (path, req, req, ", ".join(headers)))
        parse_fill, fill_mode = fill_rate_parser([r.get(mapping.get("fill_rate", ""), "") for r in rows])
        if "fill_rate" in mapping:
            note("fill rate read as %s" % fill_mode)
        file_object = args.object or object_from(os.path.basename(path))
        custom_inferred = 0
        for r in rows:
            get = lambda k: (r.get(mapping[k]) if k in mapping else None)
            name = (get("name") or "").strip()
            if not name:
                continue
            obj = norm_object(get("object")) or file_object
            created_by = (get("created_by") or "").strip()
            custom = parse_bool(get("custom"))
            if custom is None and parse_bool(get("vendor_defined")) is not None:
                custom = not parse_bool(get("vendor_defined"))
            if custom is None:
                custom_inferred += 1
                custom = not (name.startswith("hs_") or name in HUBSPOT_DEFAULT_NAMES or created_by.lower() == "hubspot")
            fields.append({
                "object": obj, "name": name, "label": (get("label") or "").strip() or name,
                "type": (get("type") or "").strip(), "field_type": (get("field_type") or "").strip(),
                "type_family": type_family(get("type"), get("field_type")),
                "description": (get("description") or "").strip() if "description" in mapping else None,
                "group": (get("group") or "").strip(), "options": parse_options(get("options")),
                "custom": custom, "unique": None, "calculated": "calculation" in (get("field_type") or "").lower(),
                "fill_rate": parse_fill(get("fill_rate")) if "fill_rate" in mapping else None,
                "usages": parse_int(get("usages")) if "usages" in mapping else None,
                "created_at": (get("created_at") or "").strip() or None,
                "created_by_vendor": created_by.lower() == "hubspot" if created_by else None,
                "last_updated": (get("last_updated") or "").strip() or None,
                "update_source": (get("update_source") or "").strip() or None,
            })
        if custom_inferred:
            note("custom flag inferred for %d rows (hs_ prefix, default names, or created by HubSpot)" % custom_inferred)
        missing_obj = sum(1 for f in fields if not f["object"])
        if missing_obj:
            note("%d rows have no object: add an Object column, pass --object, or name the file after the object" % missing_obj)
    return {"fields": fields, "header_map": maps}


def walk_filters(node, out, other=None):
    """Property filters on the enrolled object go to out. An event filter names an event property such as
    hs_form_id, and an association branch names another object's property, so neither can be resolved
    against the enrolled object's fields: each such branch is counted in other and not walked."""
    if isinstance(node, dict):
        if node.get("eventTypeId") or str(node.get("filterBranchType") or "").upper() not in OBJECT_BRANCH_TYPES:
            if other is not None:
                other.append(str(node.get("eventTypeId") or node.get("filterBranchType")))
            return
        prop = node.get("property") if isinstance(node.get("property"), str) else node.get("propertyName")
        if isinstance(prop, str):
            vals = []
            op = node.get("operation") if isinstance(node.get("operation"), dict) else {}
            for src in (node, op):
                for k in ("value", "values"):
                    v = src.get(k)
                    if isinstance(v, list):
                        vals.extend(str(x) for x in v if not isinstance(x, (dict, list)))
                    elif v is not None and not isinstance(v, (dict, list)):
                        vals.append(str(v))
            out.append({"property": prop, "values": vals})
        for v in node.values():
            walk_filters(v, out, other)
    elif isinstance(node, list):
        for v in node:
            walk_filters(v, out, other)


def object_name(type_id):
    """Plural name for a standard object; any other objectTypeId stays as it is, so its fields are not looked
    up on the wrong object. None when the flow names no object."""
    t = str(type_id or "").strip()
    return HUBSPOT_OBJECT_TYPES.get(t, t) or None


def write_from(prop, val):
    """One write from a value block. An empty static value clears the field; the reference shows no clear
    example, so a value block the script cannot read is set-dynamic and LIFE-05 notes it on lifecycle."""
    if isinstance(val, dict) and val.get("staticValue") not in (None, ""):
        return {"field": prop, "op": "set", "value": str(val.get("staticValue"))}
    if val in (None, {}, "") or (isinstance(val, dict) and (
            "CLEAR" in str(val.get("type", "")).upper() or val.get("staticValue") == "")):
        return {"field": prop, "op": "clear", "value": None}
    return {"field": prop, "op": "set-dynamic", "value": None}


def first_key(d, names, default=None):
    for n in names:
        if n in d:
            return d[n]
    for k, v in d.items():
        if any(n in k.lower() for n in names):
            return v
    return default


def hubspot_flows(files, args):
    credit_ids = [x.strip() for x in (args.credit_action_ids or "").split(",") if x.strip()]
    return flows_from_data([(os.path.basename(p), read_json(p)) for p in files], credit_ids)


def flows_from_data(datasets, credit_ids=()):
    """datasets: (source name, parsed JSON) pairs. A flow with no actions key is a summary (the v4 list
    endpoint returns key fields only): it is kept with writes_known, actions_known and refs_known false, so
    the rules that need its definition read NOT CHECKED instead of passing."""
    credit = set(CREDIT_ACTIONS) | set(credit_ids)
    autos = []
    unknown_total, summaries = {}, []
    for path, data in datasets:
        flows = data.get("results", data.get("flows", [data])) if isinstance(data, dict) else data
        if not isinstance(flows, list):
            fail("%s: expected a flow object, a list of flows or an object with a results list" % path)
        for fl in flows:
            if not isinstance(fl, dict):
                continue
            full = isinstance(fl.get("actions"), list)
            if not full:
                summaries.append(str(fl.get("name") or fl.get("id") or fl.get("flowId") or "?"))
            actions = fl.get("actions") or []
            writes, owner_writes, created, types, unknown = [], [], [], {}, []
            for a in actions:
                t = str(a.get("actionTypeId") or "")
                if not t:  # branches have no actionTypeId; any other such action could write or delete anything
                    kind = str(a.get("type") or "")
                    if "BRANCH" not in kind.upper() and (kind or a.get("fields")):
                        unknown.append(kind or "no actionTypeId")
                        unknown_total[unknown[-1]] = unknown_total.get(unknown[-1], 0) + 1
                    continue
                types[t] = types.get(t, 0) + 1
                f = a.get("fields") or {}
                if t == "0-5":
                    w = write_from(f.get("property_name") or f.get("propertyName"), f.get("value"))
                    if f.get("association"):  # the action edits an associated record, not the enrolled one
                        w["associated"] = True
                    writes.append(w)
                elif t == "0-11":  # rotate-owner keys are not in the public action reference: null when absent
                    ow = first_key(f, ["overwrite_current_owner", "overwrite"], None)
                    owner_writes.append({
                        "field": first_key(f, ["target_property", "targetProperty"], "hubspot_owner_id"),
                        "overwrite": None if ow is None else bool(ow),
                        "available_only": first_key(f, ["only_available", "available"], None),
                        "rotation_users": len(first_key(f, ["user_ids", "userIds"], []) or []),
                    })
                elif t == "0-14":
                    made = object_name(f.get("object_type_id") or f.get("objectTypeId")) or "unknown"
                    created.append(made)
                    for item in f.get("properties") or []:  # a new record's values are writes too (deal stage, owner)
                        if isinstance(item, dict):
                            writes.append(dict(write_from(item.get("targetProperty"), item.get("value")), object=made))
                elif t not in HUBSPOT_ACTIONS and t not in HUBSPOT_NEUTRAL_ACTIONS and t not in credit:
                    unknown.append(t)
                    unknown_total[t] = unknown_total.get(t, 0) + 1
            crit = fl.get("enrollmentCriteria") or {}
            filters, other = [], []
            walk_filters({k: v for k, v in fl.items() if k != "actions"}, filters, other)
            for a in actions:  # branch conditions are reads too
                if not a.get("actionTypeId"):
                    walk_filters(a, filters, other)
            trig = []
            walk_filters(crit.get("reEnrollmentTriggersFilterBranches") or [], trig)
            obj = object_name(fl.get("objectTypeId"))
            if obj is None and "CONTACT" in str(fl.get("type", "")).upper():
                obj = "contacts"
            events = sorted(set(str(b.get("eventTypeId")) for b in crit.get("eventFilterBranches") or []
                                if isinstance(b, dict) and b.get("eventTypeId")))
            autos.append({
                "id": str(fl.get("id") or fl.get("flowId") or ""), "name": fl.get("name") or "",
                "platform_type": "hubspot_workflow", "layer": "workflow", "object": obj,
                "enabled": bool(fl.get("isEnabled", fl.get("enabled", False))),
                "description": fl.get("description"), "created_at": fl.get("createdAt"), "updated_at": fl.get("updatedAt"),
                "writes": [w for w in writes if w["field"]], "owner_writes": owner_writes,
                "consent_actions": sorted(t for t in types if t in CONSENT_ACTIONS),
                "destructive_actions": sorted(t for t in types if t in DESTRUCTIVE_ACTIONS),
                "credit_actions": sorted(t for t in types if t in credit),
                "sequence_enrolments": sum(types.get(t, 0) for t in SEQUENCE_ACTIONS),
                "creates_records": created,
                "reenrollment": {"enabled": bool(crit.get("shouldReEnroll", False)),
                                 "trigger_fields": sorted(set(x["property"] for x in trig)),
                                 "unread_event_triggers": [e for e in events if e == PROPERTY_CHANGE_EVENT]},
                "filter_refs": dedupe_refs(filters), "other_object_refs": len(other),
                "action_types": types, "unknown_action_types": sorted(set(unknown)),
                "writes_known": full, "actions_known": full, "refs_known": full,
            })
    if unknown_total:
        note("action types not classified (listed per workflow, not dropped): " +
             ", ".join("%s x%d" % (k, v) for k, v in sorted(unknown_total.items())))
    if summaries:
        note("summary only, no actions (writes, actions and references marked unknown): %s; read each with "
             "'workflows get <flowId>' or GET /automation/v4/flows/{flowId} and normalize that output instead"
             % ", ".join(summaries))
    return {"automations": autos}


def dedupe_refs(refs):
    seen = {}
    for r in refs:
        cur = seen.setdefault(r["property"], [])
        for v in r["values"]:
            if v not in cur:
                cur.append(v)
    return [{"property": k, "values": v} for k, v in seen.items()]


SF_FAMILY = {"string": "text", "textarea": "text", "email": "text", "phone": "text", "url": "text",
             "encryptedstring": "text", "picklist": "enum", "multipicklist": "enum", "combobox": "enum",
             "double": "number", "currency": "number", "percent": "number", "int": "number", "long": "number",
             "date": "date", "datetime": "date", "time": "date", "boolean": "bool", "reference": "reference",
             "id": "reference"}


def read_descriptions(values):
    """--descriptions [object=]file.csv: the Tooling API FieldDefinition query (QualifiedApiName, Description)
    saved as CSV, one file per object. The object comes from object=, an EntityDefinition column or the file
    name (fielddefinition-opportunity.csv). Returns (object, field) -> description, "" when blank."""
    out = {}
    for v in values or []:
        obj, path = (v.split("=", 1) if "=" in v and os.sep not in v.split("=", 1)[0] else (None, v))
        if not os.path.isfile(path):
            fail("--descriptions: no such file: %s" % path)
        rows, headers = read_csv(path)
        hm = {norm_header(h).replace(" ", ""): h for h in headers}
        if "qualifiedapiname" not in hm or "description" not in hm:
            fail("%s: --descriptions needs QualifiedApiName and Description columns (columns: %s)" % (path, ", ".join(headers)))
        ent = next((h for k, h in hm.items() if k.startswith("entitydefinition")), None)
        for r in rows:
            o = norm_object(obj) or (norm_object(r.get(ent)) if ent else None) or object_from(os.path.basename(path))
            if not o:
                fail("%s: cannot tell which object these fields belong to; pass --descriptions <object>=%s" % (path, path))
            out[(o, (r.get(hm["qualifiedapiname"]) or "").strip().lower())] = (r.get(hm["description"]) or "").strip()
    return out


def salesforce_describe(files, args):
    fields = []
    descs = read_descriptions(args.descriptions)
    for path in files:
        data = read_json(path)
        res = data.get("result", data) if isinstance(data, dict) else {}
        obj = norm_object(res.get("name")) or args.object or object_from(os.path.basename(path))
        for f in res.get("fields") or []:
            t = str(f.get("type") or "").lower()
            fields.append({
                "object": obj, "name": f.get("name"), "label": f.get("label") or f.get("name"), "type": t,
                "field_type": t, "type_family": SF_FAMILY.get(t, "other"),
                "description": descs.get((obj, str(f.get("name") or "").lower())), "group": "",
                "options": [{"label": p.get("label") or p.get("value"), "value": p.get("value"),
                             "raw": p.get("label") or p.get("value"), "active": p.get("active", True)}
                            for p in (f.get("picklistValues") or []) if p.get("active", True)],
                "custom": bool(f.get("custom")), "unique": bool(f.get("unique")),
                "external_id": bool(f.get("externalId")), "calculated": bool(f.get("calculated")),
                "fill_rate": None, "usages": None, "created_at": None,
            })
        note("%s: %d fields from object %s (fill rates come from separate counts)" % (path, len(res.get("fields") or []), obj))
    if descs:
        got = sum(1 for f in fields if f["description"] is not None)
        note("descriptions matched for %d of %d fields; fields not in the FieldDefinition files keep none" % (got, len(fields)))
    else:
        note("no descriptions: FLD-06 stays NOT CHECKED until --descriptions passes the FieldDefinition query results")
    return {"fields": fields}


def salesforce_flows(files, args):
    autos = []
    for path in files:
        rows, headers = read_csv(path)
        cols = set(norm_header(h).replace(" ", "") for h in headers)
        for col, label, why in (("triggerorder", "TriggerOrder", "AUTO-09 will warn about every object with two or more "
                                                                 "record-triggered flows of one timing, even where Trigger Order is set"),
                                ("description", "Description", "AUTO-01 cannot see flow descriptions")):
            if col not in cols:
                note("%s: no %s column, so %s; re-run the FlowDefinitionView query in references/reading-salesforce.md"
                     % (path, label, why))
        for r in rows:
            g = {norm_header(k).replace(" ", ""): v for k, v in r.items()}
            ptype = (g.get("processtype") or "").strip()
            trig = (g.get("triggertype") or "").strip()
            if ptype.lower() in ("workflow", "customevent", "invocableprocess"):  # the three Process Builder types
                layer = "process_builder"
            elif trig.lower().startswith("record"):
                layer = "flow"
            else:
                layer = "flow_other"
            autos.append({
                "id": (g.get("apiname") or "").strip(), "name": (g.get("label") or g.get("apiname") or "").strip(),
                "platform_type": "salesforce_" + layer, "layer": layer,
                "object": norm_object(g.get("triggerobjectoreventlabel")) or None, "trigger_type": trig or None,
                "trigger_order": (g.get("triggerorder") or "").strip() or None,
                "enabled": parse_bool(g.get("isactive")) is True,
                "description": (g.get("description") or "").strip() if "description" in cols else None,
                "updated_at": (g.get("lastmodifieddate") or "").strip() or None,
                "writes": [], "owner_writes": [], "consent_actions": [], "destructive_actions": [], "credit_actions": [],
                "creates_records": [], "filter_refs": [], "reenrollment": {"enabled": False, "trigger_fields": []},
                "writes_known": False, "actions_known": False, "refs_known": False,
            })
        note("%s: %d flows; field writes, actions and filters are not in FlowDefinitionView. Add them per flow in the "
             "inventory from flow metadata or Org Check, with writes_known, actions_known and refs_known set to true" % (path, len(rows)))
    return {"automations": autos}


ATTIO_FAMILY = {"checkbox": "bool", "number": "number", "currency": "number", "rating": "number", "date": "date",
                "timestamp": "date", "status": "enum", "select": "enum", "record-reference": "reference",
                "actor-reference": "reference", "text": "text", "domain": "text", "email-address": "text",
                "phone-number": "text", "personal-name": "text", "location": "text"}


def attio_attributes(files, args):
    options = {}
    if args.options:
        raw = read_json(args.options)
        for slug, vals in (raw.items() if isinstance(raw, dict) else []):
            options[slug] = [v.get("title") if isinstance(v, dict) else str(v) for v in vals
                             if not (isinstance(v, dict) and v.get("is_archived"))]
    fields = []
    for path in files:
        data = read_json(path)
        attrs = data.get("data", data) if isinstance(data, dict) else data
        obj = args.object or object_from(os.path.basename(path))
        for a in attrs or []:
            slug = a.get("api_slug")
            t = str(a.get("type") or "")
            fields.append({
                "object": obj, "name": slug, "label": a.get("title") or slug, "type": t, "field_type": t,
                "type_family": ATTIO_FAMILY.get(t, "other"), "description": a.get("description") or "",
                "group": "", "options": [{"label": o, "value": o, "raw": o} for o in options.get(slug, [])],
                "custom": not bool(a.get("is_system_attribute")), "unique": bool(a.get("is_unique")),
                "required": bool(a.get("is_required")), "writable": a.get("is_writable"),
                "archived": bool(a.get("is_archived")), "multiselect": bool(a.get("is_multiselect")),
                "calculated": False, "fill_rate": None, "usages": None, "created_at": a.get("created_at"),
            })
        note("%s: %d attributes for object %s" % (path, len(attrs or []), obj or "not set: pass --object, because "
                                                  "the attributes response names objects by ID only"))
    return {"fields": [f for f in fields if not f["archived"]]}


def generic_fields(files, args):
    fields = []
    for path in files:
        rows, headers = read_csv(path)
        hmap = {norm_header(h).replace(" ", "_"): h for h in headers}
        for req in REQUIRED["generic-fields"]:
            if req not in hmap:
                fail("%s: column %r is required (columns: %s)" % (path, req, ", ".join(headers)))
        parse_fill, _ = fill_rate_parser([r.get(hmap.get("fill_rate", ""), "") for r in rows])
        for r in rows:
            g = lambda k: (r.get(hmap[k]) if k in hmap else None)
            t = (g("type") or "").strip()
            fields.append({
                "object": norm_object(g("object")), "name": (g("name") or "").strip(),
                "label": (g("label") or g("name") or "").strip(), "type": t, "field_type": t,
                "type_family": type_family(t), "description": (g("description") or "").strip() if "description" in hmap else None,
                "group": "", "options": parse_options(g("options")), "custom": parse_bool(g("custom")),
                "unique": parse_bool(g("unique")), "calculated": False,
                "fill_rate": parse_fill(g("fill_rate")) if "fill_rate" in hmap else None,
                "usages": parse_int(g("usages")) if "usages" in hmap and (g("usages") or "").strip() else None,
                "created_at": (g("created_at") or "").strip() or None,
                "writers": [w.strip() for w in re.split(r"[;|]", g("writers") or "") if w.strip()],
            })
    return {"fields": fields}


ADAPTERS = {
    "hubspot-properties": hubspot_properties, "hubspot-flows": hubspot_flows,
    "salesforce-describe": salesforce_describe, "salesforce-flows": salesforce_flows,
    "attio-attributes": attio_attributes, "generic-fields": generic_fields,
}


def main():
    p = argparse.ArgumentParser(description="Normalize CRM definition exports for audit_crm.py (definitions only).")
    p.add_argument("adapter", choices=sorted(ADAPTERS))
    p.add_argument("files", nargs="+")
    p.add_argument("--out", help="write the fragment here (default: print to stdout)")
    p.add_argument("--object", help="object for every row when the file has no object column")
    p.add_argument("--map", action="append", help="key=Header to map a column by hand (repeatable)")
    p.add_argument("--options", help="attio-attributes: JSON of select options and statuses per attribute slug")
    p.add_argument("--credit-action-ids", help="hubspot-flows: extra action type IDs that consume credits")
    p.add_argument("--descriptions", action="append", help="salesforce-describe: [object=]file.csv from the FieldDefinition "
                                                            "query, one per object (repeatable)")
    args = p.parse_args()
    for f in args.files:
        if not os.path.isfile(f):
            fail("no such file: %s" % f)
    result = ADAPTERS[args.adapter](args.files, args)
    kind = "fields" if "fields" in result else "automations"
    if not result[kind]:
        fail("no %s found in %s: check the file and the adapter" % (kind, ", ".join(os.path.basename(f) for f in args.files)))
    out = {"generated_by": "normalize_exports.py", "adapter": args.adapter,
           "sources": [os.path.basename(f) for f in args.files]}
    out.update(result)
    text = json.dumps(out, indent=1, ensure_ascii=False)
    if args.out:
        os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
        with open(args.out, "w", encoding="utf-8") as fh:
            fh.write(text + "\n")
    else:
        print(text)
    print("%s: %d %s written%s. Definitions only; no record values were read." % (
        args.adapter, len(result[kind]), kind, (" to " + args.out) if args.out else ""), file=sys.stderr)


if __name__ == "__main__":
    main()