增加batch-gen.task.xml测试文件，在batch dsl中集成BatchGenModel支持

docs/dev-guide/report/excel-import.md

@@ -14,7 +14,7 @@
 
 ## 配置说明
 
-## 如果解析列表
+## 如何解析列表
 
 * sheet或者field上标注list=true，表示将会解析得到一个列表
 * 列表的第一列必须是数字列，不要求编号唯一，也不要求编号列在字段列表中定义。它仅仅被用于确定列表行的范围。
@@ -23,7 +23,7 @@
 
 ```xml
 <sheet name="ss" namePattern=".*" multiple="true" multipleAsMap="true" field="ss">
-    
+
 </sheet>
 ```
 
@@ -45,7 +45,7 @@
 ExcelReportHelper中提供了根据导入的业务数据自动生成html或者xlsx文件的方法。
 
 ```javascript
-    Object bean = ExcelHelper.loadXlsxObject("/nop/test/imp/test5.imp.xml", resource);    
+    Object bean = ExcelHelper.loadXlsxObject("/nop/test/imp/test5.imp.xml", resource);
     String html = ExcelReportHelper.getHtmlForXlsxObject(impModelPath, bean, scope);
     ExcelReportHelper.saveXlsxObject(impModelPath, resource, bean);
 ```
@@ -66,7 +66,7 @@ ExcelReportHelper中提供了根据导入的业务数据自动生成html或者xl
 <field name="columns" displayName="项目指标" list="true">
     <fields>
         <field name="name" displayName="指标" mandatory="true"/>
-        
+
         <field name="indexValue" displayName="X年指标" virtual="true">
             <schema stdDomain="int"/>
             <valueExpr>
@@ -88,7 +88,7 @@ ExcelReportHelper中提供了根据导入的业务数据自动生成html或者xl
 
             <xpt:valueExpr>
                 _.findWhere(cell.rp.ev.indexValues,'year',cell.cp.ev.$toInt()).value
-            </xpt:valueExpr>            
+            </xpt:valueExpr>
         </field>
 
     </fields>
@@ -201,17 +201,20 @@ ExcelReportHelper中提供了根据导入的业务数据自动生成html或者xl
 如果要使用Nop平台的Excel导入导出功能，只需要在pom文件中引入如下模块
 
 ```xml
-        <!-- 实现Nop平台与spring框架的集成，不依赖于数据库，不依赖Web环境 -->
-<dependency>
+
+<dependencies>
+  <!-- 实现Nop平台与spring框架的集成，不依赖于数据库，不依赖Web环境 -->
+  <dependency>
     <groupId>io.github.entropy-cloud</groupId>
     <artifactId>nop-spring-core-starter</artifactId>
-</dependency>
+  </dependency>
 
-        <!-- Excel解析和报表引擎支持 -->
-<dependency>
-<groupId>io.github.entropy-cloud</groupId>
-<artifactId>nop-report-core</artifactId>
-</dependency>
+  <!-- Excel解析和报表引擎支持 -->
+  <dependency>
+    <groupId>io.github.entropy-cloud</groupId>
+    <artifactId>nop-report-core</artifactId>
+  </dependency>
+</dependencies>
 
 ```
 

nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/BatchTaskBuilder.java

@@ -23,6 +23,7 @@
 import io.nop.batch.core.loader.ChunkSortBatchLoader;
 import io.nop.batch.core.loader.RetryBatchLoader;
 import io.nop.batch.core.processor.BatchChunkProcessor;
+import io.nop.batch.core.processor.BatchSequentialProcessor;
 import io.nop.batch.core.processor.InvokerBatchChunkProcessor;
 import io.nop.commons.concurrent.executor.ExecutorHelper;
 import io.nop.commons.concurrent.ratelimit.DefaultRateLimiter;
@@ -44,6 +45,7 @@ public class BatchTaskBuilder<S, R> implements IBatchTaskBuilder {
     private Long taskVersion;
     private IBatchLoaderProvider<S> loader;
     private IBatchConsumerProvider<R> consumer;
+    private boolean useBatchRequestGenerator;
     private IBatchProcessorProvider<S, R> processor;
     private int batchSize = 100;
 
@@ -133,6 +135,12 @@ public BatchTaskBuilder<S, R> taskVersion(Long taskVersion) {
         return this;
     }
 
+    @PropertySetter
+    public BatchTaskBuilder<S, R> useBatchRequestGenerator(boolean b) {
+        this.useBatchRequestGenerator = b;
+        return this;
+    }
+
     @PropertySetter
     public BatchTaskBuilder<S, R> taskKeyExpr(IEvalFunction expr) {
         this.taskKeyExpr = expr;
@@ -328,6 +336,9 @@ protected IBatchChunkProcessor buildChunkProcessor(IBatchTaskContext context) {
         if (this.processor != null) {
             // 如果设置了processor,则先执行processor再调用consumer，否则直接调用consumer
             IBatchProcessor<S, R> processor = this.processor.setup(context);
+            if (useBatchRequestGenerator) {
+                processor = new BatchSequentialProcessor(processor);
+            }
             consumer = new BatchProcessorConsumer<>(processor, (IBatchConsumer<R>) consumer);
         }
 

nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/loader/ResourceRecordLoaderProvider.java

@@ -7,6 +7,7 @@
  */
 package io.nop.batch.core.loader;
 
+import io.nop.api.core.convert.ConvertHelper;
 import io.nop.api.core.exceptions.NopException;
 import io.nop.batch.core.IBatchAggregator;
 import io.nop.batch.core.IBatchChunkContext;
@@ -15,6 +16,7 @@
 import io.nop.batch.core.IBatchTaskContext;
 import io.nop.batch.core.common.AbstractBatchResourceHandler;
 import io.nop.commons.util.IoHelper;
+import io.nop.core.lang.eval.IEvalAction;
 import io.nop.core.resource.IResource;
 import io.nop.core.resource.record.IResourceRecordInputProvider;
 import io.nop.dataset.record.IRecordInput;
@@ -50,7 +52,7 @@ public class ResourceRecordLoaderProvider<S> extends AbstractBatchResourceHandle
     /**
      * 最多读取多少行数据（包含跳过的记录）
      */
-    long maxCount;
+    IEvalAction maxCountExpr;
 
     /**
      * 跳过起始的多少行数据
@@ -111,12 +113,8 @@ public boolean isSaveState() {
         return saveState;
     }
 
-    public long getMaxCount() {
-        return maxCount;
-    }
-
-    public void setMaxCount(long maxCount) {
-        this.maxCount = maxCount;
+    public void setMaxCountExpr(IEvalAction maxCountExpr) {
+        this.maxCountExpr = maxCountExpr;
     }
 
     public long getSkipCount() {
@@ -164,8 +162,10 @@ LoaderState<S> newLoaderState(IBatchTaskContext context) {
             skip(input, skipCount, state);
         }
 
-        if (maxCount > 0) {
-            input = input.limit(maxCount);
+        if (maxCountExpr != null) {
+            Long maxCount = ConvertHelper.toLong(maxCountExpr.invoke(context));
+            if (maxCount != null)
+                input = input.limit(maxCount);
         }
 
         if (recordRowNumber) {

nop-batch/nop-batch-dsl/pom.xml

@@ -36,6 +36,11 @@
             <artifactId>nop-batch-dao</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>io.github.entropy-cloud</groupId>
+            <artifactId>nop-batch-gen</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>io.github.entropy-cloud</groupId>
             <artifactId>nop-task-core</artifactId>

nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/FileBatchSupport.java

@@ -34,8 +34,8 @@ public static IBatchLoaderProvider<Object> newFileReader(BatchFileReaderModel lo
         loader.setResourceLoader(resourceLoader);
         loader.setSaveState(Boolean.TRUE.equals(saveState));
 
-        if (loaderModel.getMaxCount() != null)
-            loader.setMaxCount(loaderModel.getMaxCount());
+        if (loaderModel.getMaxCountExpr() != null)
+            loader.setMaxCountExpr(loaderModel.getMaxCountExpr());
         loader.setPathExpr(loaderModel.getFilePath());
         loader.setEncoding(loaderModel.getEncoding());
         loader.setAggregator(aggregator);

nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/ModelBasedBatchTaskBuilderFactory.java

@@ -22,17 +22,21 @@
 import io.nop.batch.core.processor.MultiBatchProcessorProvider;
 import io.nop.batch.dsl.model.BatchChunkProcessorBuilderModel;
 import io.nop.batch.dsl.model.BatchConsumerModel;
+import io.nop.batch.dsl.model.BatchGeneratorModel;
 import io.nop.batch.dsl.model.BatchListenersModel;
 import io.nop.batch.dsl.model.BatchLoaderModel;
 import io.nop.batch.dsl.model.BatchOrmWriterModel;
 import io.nop.batch.dsl.model.BatchProcessorModel;
 import io.nop.batch.dsl.model.BatchTaggerModel;
 import io.nop.batch.dsl.model.BatchTaskModel;
+import io.nop.batch.gen.loader.BatchGenLoaderProvider;
 import io.nop.commons.collections.OrderByComparator;
 import io.nop.commons.util.CollectionHelper;
 import io.nop.commons.util.retry.IRetryPolicy;
+import io.nop.core.lang.eval.IEvalAction;
 import io.nop.core.lang.eval.IEvalFunction;
 import io.nop.core.reflect.bean.BeanTool;
+import io.nop.core.resource.VirtualFileSystem;
 import io.nop.dao.api.IDaoProvider;
 import io.nop.dao.api.INamedSqlBuilder;
 import io.nop.dao.jdbc.IJdbcTemplate;
@@ -93,6 +97,7 @@ public IBatchTaskBuilder newTaskBuilder(IBeanProvider beanContainer) {
         builder.taskKeyExpr(batchTaskModel.getTaskKeyExpr());
         builder.allowStartIfComplete(batchTaskModel.getAllowStartIfComplete());
         builder.startLimit(batchTaskModel.getStartLimit());
+        builder.useBatchRequestGenerator(batchTaskModel.isUseBatchRequestGenerator());
 
         builder.batchSize(batchTaskModel.getBatchSize());
         if (batchTaskModel.getJitterRatio() != null)
@@ -257,6 +262,8 @@ private IBatchLoaderProvider<Object> buildLoader0(BatchLoaderModel loaderModel,
             return newJdbcReader(loaderModel.getJdbcReader(), beanProvider, jdbcTemplate, sqlLibManager);
         } else if (loaderModel.getOrmReader() != null) {
             return newOrmReader(loaderModel.getOrmReader(), daoProvider);
+        } else if (loaderModel.getGenerator() != null) {
+            return newGenerator(loaderModel.getGenerator());
         } else if (loaderModel.getSource() != null) {
             return context -> (batchSize, ctx) -> (List<Object>) loaderModel.getSource().call2(null,
                     batchSize, ctx, ctx.getEvalScope());
@@ -265,6 +272,16 @@ private IBatchLoaderProvider<Object> buildLoader0(BatchLoaderModel loaderModel,
         }
     }
 
+    private IBatchLoaderProvider<Object> newGenerator(BatchGeneratorModel generatorModel) {
+        String genModelPath = generatorModel.getGenModelPath();
+        IEvalAction totalCountExpr = generatorModel.getTotalCountExpr();
+        BatchGenLoaderProvider<Object> provider = new BatchGenLoaderProvider<>();
+        provider.setTotalCount(totalCountExpr);
+        provider.setResourcePath(genModelPath);
+        provider.setResourceLoader(VirtualFileSystem.instance());
+        return provider;
+    }
+
     private IBatchAggregator<Object, Object, Map<String, Object>> loadAggregator(String beanName, IBeanProvider beanContainer) {
         if (beanName == null)
             return null;

nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/BatchGeneratorModel.java

@@ -0,0 +1,9 @@
+package io.nop.batch.dsl.model;
+
+import io.nop.batch.dsl.model._gen._BatchGeneratorModel;
+
+public class BatchGeneratorModel extends _BatchGeneratorModel{
+    public BatchGeneratorModel(){
+
+    }
+}

nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchFileReaderModel.java

@@ -53,10 +53,10 @@ public abstract class _BatchFileReaderModel extends io.nop.core.resource.compone
 
     /**
      *  
-     * xml name: maxCount
-     * 读取的最大记录数，缺省值为-1，表示不限制
+     * xml name: maxCountExpr
+     * 
      */
-    private java.lang.Long _maxCount ;
+    private io.nop.core.lang.eval.IEvalAction _maxCountExpr ;
 
     /**
      *  
@@ -176,19 +176,19 @@ public void setHeaders(java.util.Set<java.lang.String> value){
 
     /**
      * 
-     * xml name: maxCount
-     *  读取的最大记录数，缺省值为-1，表示不限制
+     * xml name: maxCountExpr
+     *  
      */
 
-    public java.lang.Long getMaxCount(){
-      return _maxCount;
+    public io.nop.core.lang.eval.IEvalAction getMaxCountExpr(){
+      return _maxCountExpr;
     }
 
 
-    public void setMaxCount(java.lang.Long value){
+    public void setMaxCountExpr(io.nop.core.lang.eval.IEvalAction value){
         checkAllowChange();
 
-        this._maxCount = value;
+        this._maxCountExpr = value;
 
     }
 
@@ -270,7 +270,7 @@ protected void outputJson(IJsonHandler out){
         out.putNotNull("filePath",this.getFilePath());
         out.putNotNull("filter",this.getFilter());
         out.putNotNull("headers",this.getHeaders());
-        out.putNotNull("maxCount",this.getMaxCount());
+        out.putNotNull("maxCountExpr",this.getMaxCountExpr());
         out.putNotNull("newRecordInputProvider",this.getNewRecordInputProvider());
         out.putNotNull("resourceIO",this.getResourceIO());
         out.putNotNull("resourceLoader",this.getResourceLoader());
@@ -290,7 +290,7 @@ protected void copyTo(BatchFileReaderModel instance){
         instance.setFilePath(this.getFilePath());
         instance.setFilter(this.getFilter());
         instance.setHeaders(this.getHeaders());
-        instance.setMaxCount(this.getMaxCount());
+        instance.setMaxCountExpr(this.getMaxCountExpr());
         instance.setNewRecordInputProvider(this.getNewRecordInputProvider());
         instance.setResourceIO(this.getResourceIO());
         instance.setResourceLoader(this.getResourceLoader());