Merge pull request #253 from forcedotcom/develop

Added concurencyMode #251
forcedotcom · Feb 1, 2021 · 7032cd9 · 7032cd9
2 parents b14e0f5 + c0946ed
commit 7032cd9
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@
 
 ```bash
 ### ------------------------------------------- ###
-### - *** Latest version: v3.7.21         *** - ###
+### - *** Latest version: v3.8.0         *** - ###
 ### ------------------------------------------- ###
 ### - *** Make sure, that you have        *** - ###
 ### - *** the latest version installed    *** - ###

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "sfdmu",
     "description": "The Salesforce Data Loader SFDX Plugin (SFDMU) will help you to populate your org (scratch / dev / sandbox / production) with data imported from another org or CSV files. It supports all important CRUD operations (Insert/Update/Upsert/Delete) also for multiple related sObjects.",
-    "version": "3.7.21",
+    "version": "3.8.0",
     "author": "Haim Knokh",
     "bugs": "https://github.com/forcedotcom/SFDX-Data-Move-Utility/issues",
     "dependencies": {

diff --git a/src/modules/components/api_engines/bulkApiV1_0Engine.ts b/src/modules/components/api_engines/bulkApiV1_0Engine.ts
@@ -34,7 +34,13 @@ export class BulkApiV1_0Engine extends ApiEngineBase implements IApiEngine {
     async createCRUDApiJobAsync(allRecords: Array<any>): Promise<IApiJobCreateResult> {
         let connection = Sfdx.createOrgConnection(this.connectionData);
         connection.bulk.pollTimeout = CONSTANTS.POLL_TIMEOUT;
-        let job = connection.bulk.createJob(this.sObjectName, this.strOperation.toLowerCase());
+        let job = connection.bulk.createJob(
+            this.sObjectName,
+            this.strOperation.toLowerCase(),
+            {
+                concurrencyMode: this.concurrencyMode
+            }
+        );
         let recordChunks = Common.chunkArray(allRecords, this.bulkApiV1BatchSize);
         let chunks = new CsvChunks().fromArrayChunks(recordChunks);
         this.apiJobCreateResult = {
@@ -81,7 +87,7 @@ export class BulkApiV1_0Engine extends ApiEngineBase implements IApiEngine {
                         jobState: "Failed",
                         errorMessage: batchInfo.stateMessage,
                         jobId: job.id,
-                        batchId: batch.id                        
+                        batchId: batch.id
                     }));
                 }
                 // ERROR RESULT
@@ -112,7 +118,7 @@ export class BulkApiV1_0Engine extends ApiEngineBase implements IApiEngine {
                                     jobState: "Failed",
                                     errorMessage: error,
                                     jobId: job.id,
-                                    batchId: batch.id                                    
+                                    batchId: batch.id
                                 }));
                             }
                             // ERROR RESULT                                                        
@@ -166,7 +172,7 @@ export class BulkApiV1_0Engine extends ApiEngineBase implements IApiEngine {
                     }
                 });
                 if (progressCallback) {
-                    if (self.numberJobRecordsFailed > 0)  {
+                    if (self.numberJobRecordsFailed > 0) {
                         // Some records are failed
                         progressCallback(new ApiInfo({
                             jobState: "JobComplete",
@@ -175,7 +181,7 @@ export class BulkApiV1_0Engine extends ApiEngineBase implements IApiEngine {
                             jobId: job.id,
                             batchId: batch.id
                         }));
-                    }                    
+                    }
                     // Progress message: operation finished
                     progressCallback(new ApiInfo({
                         jobState: "OperationFinished",

diff --git a/src/modules/models/api_models/ApiEngineBase.ts b/src/modules/models/api_models/ApiEngineBase.ts
@@ -22,6 +22,7 @@ import { IOrgConnectionData, IFieldMapping, IFieldMappingResult } from "../commo
  */
 export default class ApiEngineBase implements IApiEngine, IFieldMapping {
 
+    concurrencyMode: string;
     pollingIntervalMs: number
     bulkApiV1BatchSize: number;
     allOrNone: boolean;
@@ -62,6 +63,7 @@ export default class ApiEngineBase implements IApiEngine, IFieldMapping {
         this.sObjectName = init.sObjectName;
         this.operation = init.operation;
         this.pollingIntervalMs = init.pollingIntervalMs;
+        this.concurrencyMode = init.concurrencyMode;
         this.updateRecordId = init.updateRecordId;
         this.bulkApiV1BatchSize = init.bulkApiV1BatchSize;
         this.allOrNone = init.allOrNone;
@@ -92,7 +94,7 @@ export default class ApiEngineBase implements IApiEngine, IFieldMapping {
 
         // Create CRUD job
         await this.createCRUDApiJobAsync(allRecords);
-        
+
         // Execute CRUD job
         let resultRecords = await this.processCRUDApiJobAsync(progressCallback);
 

diff --git a/src/modules/models/api_models/helper_interfaces.ts b/src/modules/models/api_models/helper_interfaces.ts
@@ -87,6 +87,7 @@ export interface IApiEngineInitParameters {
     sObjectName: string,
     operation: OPERATION,
     pollingIntervalMs: number,
+    concurrencyMode: string,
     updateRecordId: boolean,
     targetCSVFullFilename: string,
     createTargetCSVFiles: boolean,

diff --git a/src/modules/models/job_models/migrationJobTask.ts b/src/modules/models/job_models/migrationJobTask.ts
@@ -1402,6 +1402,7 @@ export default class MigrationJobTask {
                         sObjectName: this.sObjectName,
                         operation,
                         pollingIntervalMs: this.script.pollingIntervalMs,
+                        concurrencyMode: this.script.concurrencyMode,
                         updateRecordId,
                         targetCSVFullFilename: this.data.getTargetCSVFilename(operation, targetFilenameSuffix),
                         createTargetCSVFiles: this.script.createTargetCSVFiles,
@@ -1415,6 +1416,7 @@ export default class MigrationJobTask {
                         sObjectName: this.sObjectName,
                         operation,
                         pollingIntervalMs: this.script.pollingIntervalMs,
+                        concurrencyMode: this.script.concurrencyMode,
                         updateRecordId,
                         bulkApiV1BatchSize: this.script.bulkApiV1BatchSize,
                         targetCSVFullFilename: this.data.getTargetCSVFilename(operation, targetFilenameSuffix),
@@ -1431,6 +1433,7 @@ export default class MigrationJobTask {
                 sObjectName: this.sObjectName,
                 operation,
                 pollingIntervalMs: this.script.pollingIntervalMs,
+                concurrencyMode: this.script.concurrencyMode,
                 updateRecordId,
                 allOrNone: this.script.allOrNone,
                 targetCSVFullFilename: this.data.getTargetCSVFilename(operation, targetFilenameSuffix),

diff --git a/src/modules/models/script_models/script.ts b/src/modules/models/script_models/script.ts
@@ -49,6 +49,7 @@ export default class Script {
     objects: ScriptObject[] = new Array<ScriptObject>();
 
     pollingIntervalMs: number = CONSTANTS.DEFAULT_POLLING_INTERVAL_MS;
+    concurrencyMode: "Serial" | "Parallel" = "Parallel";
     bulkThreshold: number = CONSTANTS.DEFAULT_BULK_API_THRESHOLD_RECORDS;
     bulkApiVersion: string = CONSTANTS.DEFAULT_BULK_API_VERSION;
     bulkApiV1BatchSize: number = CONSTANTS.DEFAULT_BULK_API_V1_BATCH_SIZE;