Merge pull request #256 from rdagumampan/issue-254

Issue 254

yuniql-cli/MigrationServiceFactory.cs

@@ -74,7 +74,7 @@ public IMigrationService Create(string platform)
 
         private IMigrationService CreateInternal(IDataService dataService, IBulkImportService bulkImportService)
         {
-            var directoryService = new DirectoryService();
+            var directoryService = new DirectoryService(_traceService);
             var fileService = new FileService();
             var workspaceService = new WorkspaceService(_traceService, directoryService, fileService);
             var tokenReplacementService = new TokenReplacementService(_traceService);

yuniql-cli/Program.cs

@@ -13,8 +13,9 @@ public class Program
 
         public static int Main(string[] args)
         {
-            var directoryService = new DirectoryService();
-            var traceService = new FileTraceService(directoryService);
+            var traceService = new FileTraceService();
+
+            var directoryService = new DirectoryService(traceService);
             var fileService = new FileService();
             var workspaceService = new WorkspaceService(traceService, directoryService, fileService);
 

yuniql-core/DirectoryService.cs

@@ -1,6 +1,8 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Yuniql.Extensibility;
 
 namespace Yuniql.Core
 {
@@ -9,6 +11,13 @@ namespace Yuniql.Core
     /// </summary>
     public class DirectoryService : IDirectoryService
     {
+        private ITraceService _traceService;
+
+        public DirectoryService(ITraceService traceService)
+        {
+            _traceService = traceService;
+        }
+
         ///<inheritdoc/>
         public string[] GetDirectories(string path, string searchPattern)
         {
@@ -72,9 +81,9 @@ public string[] FilterFiles(string workingPath, string environmentCode, List<str
                     .Where(x => !x.Equals(RESERVED_DIRECTORY_NAME.TRANSACTION, System.StringComparison.InvariantCultureIgnoreCase))
                     .ToList();
                 filePathParts.Reverse();
-                
-                return filePathParts.Skip(directoryPathParts.Count).Any(a => 
-                    a.StartsWith(RESERVED_DIRECTORY_NAME.PREFIX) 
+
+                return filePathParts.Skip(directoryPathParts.Count).Any(a =>
+                    a.StartsWith(RESERVED_DIRECTORY_NAME.PREFIX)
                     && !reservedDirectories.Exists(x => x.Equals(a, System.StringComparison.InvariantCultureIgnoreCase))
                 );
             });
@@ -102,6 +111,44 @@ public string[] FilterFiles(string workingPath, string environmentCode, List<str
             return sqlScriptFiles.ToArray();
         }
 
+        ///<inheritdoc/>
+        public string[] SortFiles(string scriptDirectoryPath, string environmentCode, List<string> files)
+        {
+            //we override the default os-based file names based sort-order
+            var sortManifestFilePath = Path.Combine(scriptDirectoryPath, "_sequence.ini");
+            var sortManifestExists = File.Exists(sortManifestFilePath);
+            if (sortManifestExists)
+            {
+                var sortedFiles = new List<string>();
+                var sortManifestList = File.ReadAllLines(sortManifestFilePath)
+                    .Where(f =>
+                        !string.IsNullOrEmpty(f) &&
+                        !string.IsNullOrWhiteSpace(f) &&
+                        !f.Equals(Environment.NewLine))
+                    .ToList();
+
+                _traceService.Info("A custom execution sequence manifest is detected. Scripts will run as listed in the content of _sequence.ini file. " +
+                    $"Any scripts not listed in the manifest will be skipped and will not be committed in the version where it is placed. " +
+                    $"Skipped scripts can only be executed by moving them to the next version. Expected sequence: { Environment.NewLine}" +
+                    $"{string.Join(Environment.NewLine, sortManifestList.Select(s => "  + " + s))}");
+
+                sortManifestList.ForEach(ff =>
+                {
+                    //we use the file name or relative path to check file exists in the directory being processed
+                    var file = files.FirstOrDefault(f => f.EndsWith(ff));
+                    if (null != file)
+                        sortedFiles.Add(file);
+                });
+
+                return sortedFiles.ToArray();
+            }
+            else
+            {
+                files.Sort();
+                return files.ToArray();
+            }
+        }
+
         ///<inheritdoc/>
         public string[] FilterDirectories(string workingPath, string environmentCode, List<string> directories)
         {

yuniql-core/FileTraceService.cs

@@ -10,13 +10,11 @@ namespace Yuniql.Core
     public class FileTraceService : ITraceService
     {
         private string _traceSessionId;
-        private IDirectoryService _directoryService;
 
         ///<inheritdoc/>
-        public FileTraceService(IDirectoryService directoryService)
+        public FileTraceService()
         {
             _traceSessionId = DateTime.Now.ToString("MMddyyyy-HHmmss");
-            _directoryService = directoryService;
         }
 
         ///<inheritdoc/>
@@ -39,11 +37,11 @@ public string TraceToDirectory
             set
             {
                 //when user specified location but it does not exist
-                if (value != null && !_directoryService.Exists(value))
+                if (value != null && !Directory.Exists(value))
                 {
                     Warn($"The provided trace directory does not exist. " +
                         $"Generated logs will be saved in the current directory {Environment.CurrentDirectory}.");
-                } else if (_directoryService.Exists(value))
+                } else if (Directory.Exists(value))
                 {
                     //an existing trace directory will be used
                     _traceDirectory = value;

yuniql-core/IDirectoryService.cs

@@ -38,10 +38,33 @@ public interface IDirectoryService
         /// </summary>
         string GetFileCaseInsensitive(string path, string fileName);
 
+        /// <summary>
+        /// Returns list of files in the target environment specific directory
+        /// </summary>
+        /// <param name="path"></param>
+        /// <param name="environmentCode"></param>
+        /// <param name="files"></param>
+        /// <returns></returns>
         string[] FilterFiles(string path, string environmentCode, List<string> files);
 
+        /// <summary>
+        /// Not implemented
+        /// </summary>
+        /// <param name="path"></param>
+        /// <param name="environmentCode"></param>
+        /// <param name="directories"></param>
+        /// <returns></returns>
         string[] FilterDirectories(string path, string environmentCode, List<string> directories);
 
+        /// <summary>
+        /// Return sorted files based on default sort order or using in-placed sorting manifest _manifest.ini
+        /// </summary>
+        /// <param name="path"></param>
+        /// <param name="environmentCode"></param>
+        /// <param name="files"></param>
+        /// <returns></returns>
+        string[] SortFiles(string path, string environmentCode, List<string> files);
+
         /// <summary>
         /// Wraps <see cref="Directory.CreateDirectory"/>
         /// </summary>

yuniql-core/MigrationService.RunSqlScripts.cs

@@ -32,14 +32,18 @@ public void RunVersionSqlScripts(
             string currentScriptFile = null;
             try
             {
-                //filter out scripts when environment code is used
+                //extract all sql files in the given directory
                 var sqlScriptFiles = _directoryService.GetFiles(scriptDirectory, "*.sql").ToList();
+
+                //filter out scripts when environment code is used
                 sqlScriptFiles = _directoryService.FilterFiles(workspace, environment, sqlScriptFiles).ToList();
                 _traceService.Info($"Found {sqlScriptFiles.Count} script files on {scriptDirectory}" + (sqlScriptFiles.Count > 0 ? Environment.NewLine : string.Empty) +
                        $"{string.Join(Environment.NewLine, sqlScriptFiles.Select(s => "  + " + new FileInfo(s).Name))}");
 
-                //execute all script files in the version folder, we also make sure its sorted by file name
-                sqlScriptFiles.Sort();
+                //we also make sure its sorted by file name or using an in-place sort order manifest in the directory
+                sqlScriptFiles = _directoryService.SortFiles(scriptDirectory, environment, sqlScriptFiles).ToList();
+
+                //execute all script files in the version folder
                 sqlScriptFiles.ForEach(scriptFile =>
                 {
                     currentScriptFile = scriptFile;
@@ -121,7 +125,7 @@ public void RunNonVersionSqlScripts(
             bool isRequiredClearedDraft = false
             )
         {
-            //extract and filter out scripts when environment code is used
+            //extract all sql files in the given directory
             var sqlScriptFiles = _directoryService.GetAllFiles(workspace, "*.sql").ToList();
 
             // Throw exception when --require-cleared-draft is set to TRUE 
@@ -131,12 +135,15 @@ public void RunNonVersionSqlScripts(
                     $"Move the script files to a version directory and re-execute the migration. Or remove --require-cleared-draft in parameter.");
             }
 
+            //filter out scripts when environment code is used
             sqlScriptFiles = _directoryService.FilterFiles(workspace, environment, sqlScriptFiles).ToList();
             _traceService.Info($"Found {sqlScriptFiles.Count} script files on {workspace}" + (sqlScriptFiles.Count > 0 ? Environment.NewLine : string.Empty) +
                    $"{string.Join(Environment.NewLine, sqlScriptFiles.Select(s => "  + " + new FileInfo(s).Name))}");
 
-            //execute all script files in the target folder
-            sqlScriptFiles.Sort();
+            //we also make sure its sorted by file name or using an in-place sort order manifest in the directory
+            sqlScriptFiles = _directoryService.SortFiles(workspace, environment, sqlScriptFiles).ToList();
+
+            //execute all script files in the version folder
             sqlScriptFiles.ForEach(scriptFile =>
             {
                 var sqlStatementRaw = _fileService.ReadAllText(scriptFile);
@@ -185,12 +192,18 @@ public void RunBulkImportScripts(
             List<KeyValuePair<string, string>> tokens = null
         )
         {
-            //extract and filter out scripts when environment code is used
+            //extract all sql files in the given directory
             var bulkFiles = _directoryService.GetFiles(scriptDirectory, "*.csv").ToList();
+
+            //filter out scripts when environment code is used
             bulkFiles = _directoryService.FilterFiles(workspace, environment, bulkFiles).ToList();
             _traceService.Info($"Found {bulkFiles.Count} bulk files on {scriptDirectory}" + (bulkFiles.Count > 0 ? Environment.NewLine : string.Empty) +
                    $"{string.Join(Environment.NewLine, bulkFiles.Select(s => "  + " + new FileInfo(s).Name))}");
-            bulkFiles.Sort();
+
+            //we also make sure its sorted by file name or using an in-place sort order manifest in the directory
+            bulkFiles = _directoryService.SortFiles(workspace, environment, bulkFiles).ToList();
+
+            //upload all bulk files in the version folder
             bulkFiles.ForEach(csvFile =>
             {
                 _bulkImportService.Run(connection, transaction, csvFile, bulkSeparator, bulkBatchSize: bulkBatchSize, commandTimeout: commandTimeout, tokens: tokens);

yuniql-core/MigrationServiceFactory.cs

@@ -24,7 +24,7 @@ public IMigrationService Create(IDataService dataService, IBulkImportService bul
 
         private IMigrationService CreateInternal(IDataService dataService, IBulkImportService bulkImportService)
         {
-            var directoryService = new DirectoryService();
+            var directoryService = new DirectoryService(_traceService);
             var fileService = new FileService();
             var workspaceService = new WorkspaceService(_traceService, directoryService, fileService);
             var tokenReplacementService = new TokenReplacementService(_traceService);