feat(core,schema): allow to set primary for outputFields

packages/core/src/app-middlewares/after/checks.js

@@ -15,6 +15,7 @@ const checkOutput = (output) => {
   const input = output.input || {};
   const _zapier = input._zapier || {};
   const event = _zapier.event || {};
+  const compiledApp = _zapier.app || {};
 
   const runChecks =
     event.method && event.command === 'execute' && !_zapier.skipChecks;
@@ -26,12 +27,12 @@ const checkOutput = (output) => {
       .filter((check) => {
         return (
           !bundleSkipChecks.includes(check.name) &&
-          check.shouldRun(event.method, event.bundle)
+          check.shouldRun(event.method, event.bundle, compiledApp)
         );
       })
       .map((check) => {
         return check
-          .run(event.method, output.results)
+          .run(event.method, output.results, compiledApp)
           .map((err) => ({ name: check.name, error: err }));
       });
     const checkResults = _.flatten(rawResults);

packages/core/src/checks/index.js

@@ -5,7 +5,7 @@ module.exports = {
 
   triggerIsArray: require('./trigger-is-array'),
   triggerIsObject: require('./trigger-is-object'),
-  triggerHasUniqueIds: require('./trigger-has-unique-ids'),
+  triggerHasUniquePrimary: require('./trigger-has-unique-primary'),
   triggerHasId: require('./trigger-has-id'),
   firehoseSubscriptionIsArray: require('./firehose_is_array'),
   firehoseSubscriptionKeyIsString: require('./firehose_is_string'),

packages/core/src/checks/is-trigger.js

@@ -1,5 +1,6 @@
 module.exports = (method) => {
   return (
+    // `method` will never start with "resources.". Seems like legacy code.
     (method.startsWith('triggers.') && method.endsWith('.operation.perform')) ||
     (method.startsWith('resources.') &&
       method.endsWith('.list.operation.perform'))

packages/core/src/checks/trigger-has-id.js

@@ -10,9 +10,47 @@ const isTrigger = require('./is-trigger');
 */
 const triggerHasId = {
   name: 'triggerHasId',
-  shouldRun: (method, bundle) => {
+  shouldRun: (method, bundle, compiledApp) => {
     // Hooks will have a bundle.cleanedRequest and we don't need to check they've got an id
-    return isTrigger(method) && !bundle.cleanedRequest;
+    if (!isTrigger(method) || bundle.cleanedRequest) {
+      return false;
+    }
+
+    const triggerKey = method.split('.', 2)[1];
+    if (!triggerKey) {
+      // Unreachable, but just in case
+      return false;
+    }
+
+    const outputFields = _.get(compiledApp, [
+      'triggers',
+      triggerKey,
+      'operation',
+      'outputFields',
+    ]);
+
+    if (!outputFields || !Array.isArray(outputFields)) {
+      // Unreachable, but just in case
+      return false;
+    }
+
+    // This check is only necessary if either:
+    // - field.primary not set for all fields
+    // - field.primary is set for `id` field
+    let hasPrimary = false;
+    for (const field of outputFields) {
+      if (!field) {
+        continue; // just in case
+      }
+      if (field.primary) {
+        if (field.key === 'id') {
+          return true;
+        } else {
+          hasPrimary = true;
+        }
+      }
+    }
+    return !hasPrimary;
   },
   run: (method, results) => {
     const missingIdResult = _.find(results, (result) => {

packages/core/src/checks/trigger-has-unique-primary.js

@@ -0,0 +1,99 @@
+'use strict';
+
+const _ = require('lodash');
+
+const isTrigger = require('./is-trigger');
+
+const getPreferredPrimaryKeys = (compiledApp, triggerKey) => {
+  const defaultPrimaryKeys = ['id'];
+
+  if (!triggerKey) {
+    return defaultPrimaryKeys;
+  }
+
+  const outputFields = _.get(compiledApp, [
+    'triggers',
+    triggerKey,
+    'operation',
+    'outputFields',
+  ]);
+
+  if (!outputFields || !Array.isArray(outputFields)) {
+    return defaultPrimaryKeys;
+  }
+
+  const primaryKeys = outputFields
+    .filter((f) => f && f.primary && f.key)
+    .map((f) => f.key);
+
+  return primaryKeys.length > 0 ? primaryKeys : defaultPrimaryKeys;
+};
+
+const isPrimitive = (v) => {
+  return (
+    v === null ||
+    v === undefined ||
+    typeof v === 'string' ||
+    typeof v === 'number' ||
+    typeof v === 'boolean'
+  );
+};
+
+// Gets array v where v[i] === result[primaryKeys[i]] and stringifies v into a string.
+// Throws TypeError if any of the values are not primitive.
+const stringifyValuesFromPrimaryKeys = (result, primaryKeys) => {
+  const values = primaryKeys.map((key) => result[key]);
+  return values
+    .map((v, i) => {
+      const fieldKey = primaryKeys[i];
+      if (!isPrimitive(v)) {
+        throw new TypeError(
+          `As a primary key, field "${fieldKey}" must be a primitive (non-object like number or string)`
+        );
+      }
+      if (v == null) {
+        return '';
+      }
+      return `${fieldKey}=${v.toString()}`;
+    })
+    .join('&');
+};
+
+/*
+  Makes sure the results all have a unique primary key in them.
+*/
+const triggerHasUniquePrimary = {
+  name: 'triggerHasUniquePrimary',
+  shouldRun: isTrigger,
+  run: (method, results, compiledApp) => {
+    const triggerKey = method.split('.', 2)[1];
+    const primaryKeys = getPreferredPrimaryKeys(compiledApp, triggerKey);
+
+    const idCount = {};
+
+    for (const result of results) {
+      if (result == null) {
+        // this'll get caught elsewhere, but we don't want to blow up this check
+        continue;
+      }
+
+      let id;
+      try {
+        id = stringifyValuesFromPrimaryKeys(result, primaryKeys);
+      } catch (e) {
+        return [e.message];
+      }
+
+      const count = (idCount[id] = (idCount[id] || 0) + 1);
+      if (count > 1) {
+        return [
+          `Got a two or more results with primary key of "${id}", primary key is supposed to be unique`,
+        ];
+      }
+    }
+
+    return [];
+  },
+};
+
+module.exports = triggerHasUniquePrimary;

packages/schema/docs/build/schema.md

@@ -955,6 +955,7 @@ Key | Required | Type | Description
 `required` | no | `boolean` | If this value is required or not.
 `placeholder` | no | `string` | An example value that is not saved.
 `default` | no | `string` | A default value that is saved the first time a Zap is created.
+`primary` | no | `boolean` | Use this field as part of the primary key for deduplication. You can set multiple fields as "primary", provided they are unique together. If no fields are set, Zapier will default to using the `id` field. `primary` only makes sense for `outputFields`; it will be ignored if set in `inputFields`. It only works in static `outputFields`; will not work in custom/dynamic `outputFields`. For more information, see [How deduplication works in Zapier](https://platform.zapier.com/build/deduplication).
 `dynamic` | no | [/RefResourceSchema](#refresourceschema) | A reference to a trigger that will power a dynamic dropdown.
 `search` | no | [/RefResourceSchema](#refresourceschema) | A reference to a search that will guide the user to add a search step to populate this field when creating a Zap.
 `choices` | no | [/FieldChoicesSchema](#fieldchoicesschema) | An object of machine keys and human values to populate a static dropdown.

packages/schema/exported-schema.json

@@ -201,6 +201,10 @@
           "type": "string",
           "minLength": 1
         },
+        "primary": {
+          "description": "Use this field as part of the primary key for deduplication. You can set multiple fields as \"primary\", provided they are unique together. If no fields are set, Zapier will default to using the `id` field. `primary` only makes sense for `outputFields`; it will be ignored if set in `inputFields`. It only works in static `outputFields`; will not work in custom/dynamic `outputFields`. For more information, see [How deduplication works in Zapier](https://platform.zapier.com/build/deduplication).",
+          "type": "boolean"
+        },
         "dynamic": {
           "description": "A reference to a trigger that will power a dynamic dropdown.",
           "$ref": "/RefResourceSchema"

packages/schema/lib/schemas/FieldSchema.js

@@ -79,6 +79,11 @@ module.exports = makeSchema(
         type: 'string',
         minLength: 1,
       },
+      primary: {
+        description:
+          'Use this field as part of the primary key for deduplication. You can set multiple fields as "primary", provided they are unique together. If no fields are set, Zapier will default to using the `id` field. `primary` only makes sense for `outputFields`; it will be ignored if set in `inputFields`. It only works in static `outputFields`; will not work in custom/dynamic `outputFields`. For more information, see [How deduplication works in Zapier](https://platform.zapier.com/build/deduplication).',
+        type: 'boolean',
+      },
       dynamic: {
         description:
           'A reference to a trigger that will power a dynamic dropdown.',
@@ -121,7 +126,8 @@ module.exports = makeSchema(
         type: 'boolean',
       },
       steadyState: {
-        description: 'Prevents triggering on new output until all values for fields with this property remain unchanged for 2 polls. It can be used to, e.g., not trigger on a new contact until the contact has completed typing their name. NOTE that this only applies to the `outputFields` of polling triggers.',
+        description:
+          'Prevents triggering on new output until all values for fields with this property remain unchanged for 2 polls. It can be used to, e.g., not trigger on a new contact until the contact has completed typing their name. NOTE that this only applies to the `outputFields` of polling triggers.',
         type: 'boolean',
       },
       inputFormat: {